@openlines/opl-pi-sht 0.1.16 → 0.1.18
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +8 -8
- package/configs/opl-simplebench.json.sample +20 -6
- package/extensions/opl-browser/README.md +13 -3
- package/extensions/opl-browser/browser.ts +9 -0
- package/extensions/opl-browser/extract.ts +36 -0
- package/extensions/opl-browser/index.ts +3 -3
- package/extensions/opl-browser/package-lock.json +315 -3
- package/extensions/opl-browser/package.json +5 -2
- package/extensions/opl-browser/turndown.d.ts +1 -0
- package/extensions/opl-simplebench/README.md +25 -11
- package/extensions/opl-simplebench/benchmark.ts +8 -1
- package/extensions/opl-simplebench/index.ts +42 -16
- package/extensions/opl-simplebench/types.ts +1 -1
- package/extensions/opl-simplebench/util/config.ts +11 -0
- package/package.json +2 -2
package/README.md
CHANGED
|
@@ -11,8 +11,8 @@ A portable collection of various Pi coding agent extensions. Repository director
|
|
|
11
11
|
Install a versioned release from npm or GitHub. Both ship identical content; pick one source per machine, because Pi treats the npm and Git entries as separate packages and installing both loads every extension twice.
|
|
12
12
|
|
|
13
13
|
```bash
|
|
14
|
-
pi install npm:@openlines/opl-pi-sht@0.1.
|
|
15
|
-
pi install git:github.com/linellazatin/opl-pi-sht@v0.1.
|
|
14
|
+
pi install npm:@openlines/opl-pi-sht@0.1.18
|
|
15
|
+
pi install git:github.com/linellazatin/opl-pi-sht@v0.1.18
|
|
16
16
|
```
|
|
17
17
|
|
|
18
18
|
> Omitting the version on the npm source tracks the latest published release; Git refs stay pinned, so move them with `pi install ...@v<new>`.
|
|
@@ -34,7 +34,7 @@ npx playwright install chromium
|
|
|
34
34
|
{
|
|
35
35
|
"packages": [
|
|
36
36
|
{
|
|
37
|
-
"source": "npm:@openlines/opl-pi-sht@0.1.
|
|
37
|
+
"source": "npm:@openlines/opl-pi-sht@0.1.18",
|
|
38
38
|
"extensions": ["extensions/opl-init/index.ts", "extensions/opl-todo/index.ts"]
|
|
39
39
|
}
|
|
40
40
|
]
|
|
@@ -64,7 +64,7 @@ Copy mode overwrites matching destinations. Link mode skips existing destination
|
|
|
64
64
|
| [`opl-init`](extensions/opl-init/README.md) | Fingerprinted repository-guide generator. | `/init`; no config. |
|
|
65
65
|
| [`opl-simplebench`](extensions/opl-simplebench/README.md) | Auditable provider-aware model benchmark with JSON artifacts and metrics. | `/simplebench`, `simplebench`; supports Ollama, OpenAI-compatible providers, and Bedrock; optional `opl-simplebench.json`. |
|
|
66
66
|
| [`opl-webaccess`](extensions/opl-webaccess/README.md) | Search plus readable URL/PDF retrieval with session recovery. | `web_search`, `fetch_content`, `get_search_content`; `opl-webaccess.json`. |
|
|
67
|
-
| [`opl-browser`](extensions/opl-browser/README.md) | Chromium automation via Playwright; single dispatcher tool replacing the chrome-devtools MCP.
|
|
67
|
+
| [`opl-browser`](extensions/opl-browser/README.md) | Chromium automation via Playwright with structured extraction of rendered pages; single dispatcher tool replacing the chrome-devtools MCP. | `browser` (action-based); `opl-browser.json`. |
|
|
68
68
|
| [`opl-ctxtrim`](extensions/opl-ctxtrim/README.md) | Trims verbose`ctx_*` tool-schema descriptions on outbound provider requests (~67% smaller schema, ~4,700-6,300 tokens/request). Built specifically for the [context-mode](https://github.com/mksglu/context-mode) extension. | No commands/tools; no config. |
|
|
69
69
|
| [`opl-todo`](extensions/opl-todo/README.md) | Branch-aware task tool, overlay, and task list. | `todo`, `/todos`; `opl-todo.json`. |
|
|
70
70
|
| [`opl-questionnaire`](extensions/opl-questionnaire/README.md) | Interactive structured-choice tool. | `questionnaire`; no config. |
|
|
@@ -103,7 +103,7 @@ Cold prompt-cache write, measured /init session (opl-modes lazy tools + MCP adap
|
|
|
103
103
|
|
|
104
104
|
- **`opl-init`** writes a fingerprinted `AGENTS.md` so any agent starts already oriented in a repo. It writes a baseline before asking the model to refine the guide, so smaller models cannot leave the update only in chat; it re-runs only when the tree actually changes.
|
|
105
105
|

|
|
106
|
-
- **`opl-browser`** gives full Chromium automation (navigate, snapshot, interact, screenshot, console/network capture, evaluate) through a single tool, with handle+preview output for large results.
|
|
106
|
+
- **`opl-browser`** gives full Chromium automation (navigate, snapshot, extract rendered-page markdown, interact, screenshot, console/network capture, evaluate) through a single tool, with handle+preview output for large results.
|
|
107
107
|
- **`opl-webaccess`** adds provider-backed search plus readable URL and PDF extraction, with session recovery of earlier results.
|
|
108
108
|
- **`opl-simplebench`** benchmarks models on deterministic closed-answer contracts, instruction-following, and tool-call generation so you pick a model on evidence, not vibes.
|
|
109
109
|

|
|
@@ -142,7 +142,7 @@ Installing extensions adds tool schemas (name + description + JSON parameters) t
|
|
|
142
142
|
|
|
143
143
|
| Extension | Adds to resting prompt | Est. tokens |
|
|
144
144
|
|---|---|---|
|
|
145
|
-
| `opl-browser` | `browser` tool schema | ~
|
|
145
|
+
| `opl-browser` | `browser` tool schema | ~575 |
|
|
146
146
|
| `opl-questionnaire` | `questionnaire` schema + prompt guidelines | ~532 |
|
|
147
147
|
| `opl-webaccess` | `web_search`, `fetch_content`, `get_search_content` | ~394 |
|
|
148
148
|
| `opl-simplebench` | `simplebench` schema | ~230 |
|
|
@@ -200,7 +200,7 @@ npx playwright install chromium
|
|
|
200
200
|
|
|
201
201
|
A Pi package (npm or Git) still needs the one-time `npx playwright install chromium` command shown above.
|
|
202
202
|
|
|
203
|
-
`opl-simplebench` writes a full JSON benchmark artifact in Pi's current working directory by default. `--test-all` additionally writes `research.md` and `page.html` beside `result.json` in a result bundle. `/simplebench --sequence` runs a templated multi-run protocol (for example a model warm-up curve) defined in the `runSequence` block, with per-iteration tags, optional llama-server/llamagputop metrics, and a configurable pause between iterations. Copy `configs/opl-simplebench.json.sample` to `~/.pi/agent/configs/opl-simplebench.json` to configure DDGS/SearXNG research, optional llama-server/llamagputop metadata endpoints, and the run sequence. Use `/simplebench --no-artifact` or `simplebench({ no_artifact: true })` when responses must not be written to disk. Provider credentials remain outside tracked configuration; configure them through Pi provider settings, environment variables, or Pi authentication.
|
|
203
|
+
`opl-simplebench` writes a full JSON benchmark artifact in Pi's current working directory by default. `--test-all` additionally writes `research.md` and `page.html` beside `result.json` in a result bundle. `/simplebench --sequence[=<name>]` runs a named templated multi-run protocol (for example a model warm-up curve) defined in the `runSequence.sequences` block, with per-iteration tags, optional llama-server/llamagputop metrics, and a configurable pause between iterations; block-level `llamaMetrics`/`pauseMs` are defaults a profile can override. Copy `configs/opl-simplebench.json.sample` to `~/.pi/agent/configs/opl-simplebench.json` to configure DDGS/SearXNG research, optional llama-server/llamagputop metadata endpoints, and the run sequence. Use `/simplebench --no-artifact` or `simplebench({ no_artifact: true })` when responses must not be written to disk. Provider credentials remain outside tracked configuration; configure them through Pi provider settings, environment variables, or Pi authentication.
|
|
204
204
|
|
|
205
205
|
## Tests
|
|
206
206
|
|
|
@@ -208,4 +208,4 @@ A Pi package (npm or Git) still needs the one-time `npx playwright install chrom
|
|
|
208
208
|
npm test
|
|
209
209
|
```
|
|
210
210
|
|
|
211
|
-
Run one extension suite with `npm run test:opl-<name>` for `footer`, `init`, `input`, `modes`, `questionnaire`, `todo`, `webaccess`, `simplebench`, or `ctxtrim`. Every helper, functional, and selected-entrypoint smoke check uses Bun's named-test reporter; output includes per-test status, timings, and pass/fail totals. Functional tests cover deterministic helpers where practical; smoke tests bundle entrypoints and parse config. They do not test live TUI behavior, provider credentials, network access, or PDF extraction.
|
|
211
|
+
Run one extension suite with `npm run test:opl-<name>` for `browser`, `footer`, `init`, `input`, `modes`, `questionnaire`, `todo`, `webaccess`, `simplebench`, or `ctxtrim`. Every helper, functional, and selected-entrypoint smoke check uses Bun's named-test reporter; output includes per-test status, timings, and pass/fail totals. Functional tests cover deterministic helpers where practical; smoke tests bundle entrypoints and parse config. They do not test live TUI behavior, provider credentials, network access, or PDF extraction.
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
{
|
|
2
|
-
"_comment": "Copy to ~/.pi/agent/configs/opl-simplebench.json. Endpoints are optional and only used by their corresponding Simplebench flags. runSequence powers /simplebench --sequence: each
|
|
2
|
+
"_comment": "Copy to ~/.pi/agent/configs/opl-simplebench.json. Endpoints are optional and only used by their corresponding Simplebench flags. runSequence powers /simplebench --sequence[=<name>]: each profile has a single-word name and an iterations list of flag strings; llamaMetrics appends --llama-server --llamagputop to every entry and pauseMs sleeps between iterations. Block-level llamaMetrics/pauseMs are the defaults; a profile may override either. A bare --sequence runs the only defined profile and otherwise lists the names. The legacy flat runSequence.sequence array is still accepted as an anonymous profile.",
|
|
3
3
|
"researchSearchProvider": "ddgs",
|
|
4
4
|
"researchSearchUrl": "http://127.0.0.1:8000",
|
|
5
5
|
"researchMaxResults": 5,
|
|
@@ -9,11 +9,25 @@
|
|
|
9
9
|
"enabled": false,
|
|
10
10
|
"llamaMetrics": true,
|
|
11
11
|
"pauseMs": 5000,
|
|
12
|
-
"
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
12
|
+
"sequences": [
|
|
13
|
+
{
|
|
14
|
+
"name": "warmup",
|
|
15
|
+
"iterations": [
|
|
16
|
+
"--coding-lite --tag=coldest",
|
|
17
|
+
"--3ptest --tag=colder",
|
|
18
|
+
"--coding-lite --tag=semiwarm",
|
|
19
|
+
"--test-all --research-live --tag=warm"
|
|
20
|
+
]
|
|
21
|
+
},
|
|
22
|
+
{
|
|
23
|
+
"name": "coding-only",
|
|
24
|
+
"llamaMetrics": false,
|
|
25
|
+
"pauseMs": 3000,
|
|
26
|
+
"iterations": [
|
|
27
|
+
"--coding-lite --tag=coldest",
|
|
28
|
+
"--coding-lite --tag=warmer"
|
|
29
|
+
]
|
|
30
|
+
}
|
|
17
31
|
]
|
|
18
32
|
}
|
|
19
33
|
}
|
|
@@ -14,6 +14,8 @@ No slash commands. The extension registers a single tool, `browser`, chosen by a
|
|
|
14
14
|
```ts
|
|
15
15
|
browser({ action: "navigate", url: "https://example.com" })
|
|
16
16
|
browser({ action: "snapshot" })
|
|
17
|
+
browser({ action: "extract" }) // rendered-page markdown
|
|
18
|
+
browser({ action: "extract", selector: "#about-gcash" }) // one element, verbatim
|
|
17
19
|
browser({ action: "click", selector: "button.login" })
|
|
18
20
|
browser({ action: "fill", selector: "#email", text: "a@b.com" })
|
|
19
21
|
browser({ action: "evaluate", script: "document.title" })
|
|
@@ -25,9 +27,9 @@ browser({ action: "close" })
|
|
|
25
27
|
```
|
|
26
28
|
|
|
27
29
|
Full action set: `navigate` (url, or `back`/`forward`/`reload`), `snapshot`,
|
|
28
|
-
`screenshot`, `click`, `fill`, `hover`, `press`, `select`, `evaluate`,
|
|
29
|
-
`network`, `wait_for`, `pages`, `new_page`, `select_page`, `close_page`,
|
|
30
|
-
`get`, `close`.
|
|
30
|
+
`extract`, `screenshot`, `click`, `fill`, `hover`, `press`, `select`, `evaluate`,
|
|
31
|
+
`console`, `network`, `wait_for`, `pages`, `new_page`, `select_page`, `close_page`,
|
|
32
|
+
`resize`, `get`, `close`.
|
|
31
33
|
|
|
32
34
|
## Extension features
|
|
33
35
|
|
|
@@ -37,6 +39,12 @@ Full action set: `navigate` (url, or `back`/`forward`/`reload`), `snapshot`,
|
|
|
37
39
|
evaluate output) are kept out of context: the tool returns a truncated preview
|
|
38
40
|
plus a `responseId`; call `action: "get"` with that id for the full text.
|
|
39
41
|
Screenshots are written to a file, never inlined as base64.
|
|
42
|
+
- **Structured extraction.** `extract` runs Readability + Turndown over the
|
|
43
|
+
*rendered* (post-JS) DOM — the complement to `opl-webaccess`'s `fetch_content`,
|
|
44
|
+
which only sees raw HTTP HTML. Static pages: `fetch_content`; rendered or
|
|
45
|
+
interacted-with pages: `browser:extract`. With a `selector`, the matched element
|
|
46
|
+
is converted verbatim (no article detection): Readability's candidate scoring is
|
|
47
|
+
a whole-document heuristic and mispicks inside small subtrees.
|
|
40
48
|
- **Real Chromium via Playwright.** Navigation with `domcontentloaded` waits,
|
|
41
49
|
CSS-selector interaction, viewport control, multi-page management.
|
|
42
50
|
- **Per-page capture.** Console messages and network requests are buffered per
|
|
@@ -74,6 +82,8 @@ index.ts Pi wiring: registers the single `browser` tool, TTL result store,
|
|
|
74
82
|
preview/handle logic, and session_shutdown cleanup.
|
|
75
83
|
browser.ts Playwright driver: browser/context/page lifecycle, per-page console
|
|
76
84
|
and network buffers, and the action switch.
|
|
85
|
+
extract.ts Pure rendered-HTML → markdown pipeline (linkedom + Readability +
|
|
86
|
+
turndown), duplicated from opl-webaccess to keep installs independent.
|
|
77
87
|
config.ts DEFAULT_CONFIG + loadUserConfig (user overrides win via ??).
|
|
78
88
|
```
|
|
79
89
|
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { chromium, type Browser, type BrowserContext, type Page, type ConsoleMessage } from "playwright";
|
|
2
|
+
import { extractMarkdown } from "./extract.js";
|
|
2
3
|
import type { BrowserConfig } from "./config.js";
|
|
3
4
|
|
|
4
5
|
// ponytail: single module-level Chromium instance reused across tool calls.
|
|
@@ -87,6 +88,14 @@ export async function runAction(p: BrowserParams, cfg: BrowserConfig): Promise<B
|
|
|
87
88
|
const tree = await page().locator("body").ariaSnapshot();
|
|
88
89
|
return { text: tree || "(empty snapshot)" };
|
|
89
90
|
}
|
|
91
|
+
case "extract": {
|
|
92
|
+
const scoped = Boolean(p.selector);
|
|
93
|
+
const html = scoped
|
|
94
|
+
? await page().locator(p.selector).evaluate((el: Element) => el.outerHTML)
|
|
95
|
+
: await page().content();
|
|
96
|
+
const { markdown } = extractMarkdown(html, { raw: scoped });
|
|
97
|
+
return { text: markdown || "(empty extraction)" };
|
|
98
|
+
}
|
|
90
99
|
case "screenshot": {
|
|
91
100
|
const file = p.path ?? `opl-browser-${Date.now()}.png`;
|
|
92
101
|
await page().screenshot({ path: file, fullPage: p.fullPage ?? false });
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
import { Readability } from "@mozilla/readability";
|
|
2
|
+
import { parseHTML } from "linkedom";
|
|
3
|
+
import TurndownService from "turndown";
|
|
4
|
+
|
|
5
|
+
// Structured extraction of a rendered HTML string (post-JS DOM from Playwright).
|
|
6
|
+
// Pipeline mirrors opl-webaccess/extract.ts on purpose; duplicated ~40 lines to
|
|
7
|
+
// keep per-extension installs independent (see research/browser-structured-extraction-plan.md).
|
|
8
|
+
|
|
9
|
+
const td = new TurndownService({
|
|
10
|
+
headingStyle: "atx",
|
|
11
|
+
codeBlockStyle: "fenced",
|
|
12
|
+
});
|
|
13
|
+
|
|
14
|
+
export interface ExtractedMarkdown {
|
|
15
|
+
title: string;
|
|
16
|
+
markdown: string;
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
export function extractMarkdown(html: string, opts?: { raw?: boolean }): ExtractedMarkdown {
|
|
20
|
+
// Readability's candidate scoring is a whole-document heuristic; on a small
|
|
21
|
+
// selector-scoped fragment it drops siblings. Pass raw:true for fragments.
|
|
22
|
+
if (opts?.raw) {
|
|
23
|
+
return { title: "", markdown: td.turndown(html) };
|
|
24
|
+
}
|
|
25
|
+
const { document } = parseHTML(html);
|
|
26
|
+
let article = null;
|
|
27
|
+
try {
|
|
28
|
+
article = new Readability(document as unknown as Document).parse();
|
|
29
|
+
} catch {
|
|
30
|
+
// Readability failed — fall back to raw turndown
|
|
31
|
+
}
|
|
32
|
+
if (!article?.content) {
|
|
33
|
+
return { title: "", markdown: td.turndown(html) };
|
|
34
|
+
}
|
|
35
|
+
return { title: article.title ?? "", markdown: td.turndown(article.content) };
|
|
36
|
+
}
|
|
@@ -19,7 +19,7 @@ function store(action: string, text: string): string {
|
|
|
19
19
|
}
|
|
20
20
|
|
|
21
21
|
const ACTIONS =
|
|
22
|
-
"navigate|snapshot|screenshot|click|fill|hover|press|select|evaluate|console|network|wait_for|pages|new_page|select_page|close_page|resize|get|close";
|
|
22
|
+
"navigate|snapshot|extract|screenshot|click|fill|hover|press|select|evaluate|console|network|wait_for|pages|new_page|select_page|close_page|resize|get|close";
|
|
23
23
|
|
|
24
24
|
export default function (pi: ExtensionAPI) {
|
|
25
25
|
const cfg = loadUserConfig();
|
|
@@ -33,14 +33,14 @@ export default function (pi: ExtensionAPI) {
|
|
|
33
33
|
label: "Browser",
|
|
34
34
|
description:
|
|
35
35
|
`Drive a real Chromium browser (Playwright) for web testing and inspection. One dispatcher tool; pick action: ${ACTIONS}. ` +
|
|
36
|
-
"navigate(url|back|forward|reload); snapshot (accessibility tree); screenshot (saved to file, not inlined); " +
|
|
36
|
+
"navigate(url|back|forward|reload); snapshot (accessibility tree); extract (rendered-page markdown, optionally scoped to selector); screenshot (saved to file, not inlined); " +
|
|
37
37
|
"click/fill/hover/select by CSS selector; press(key); evaluate(script in page); console/network (captured for active page); " +
|
|
38
38
|
"wait_for(selector|text); pages/new_page/select_page/close_page; resize(width,height); get(responseId) to retrieve a stored large result; close to shut the browser. " +
|
|
39
39
|
"Large outputs return a preview + responseId; call action:get with that id for the full text.",
|
|
40
40
|
parameters: Type.Object({
|
|
41
41
|
action: Type.String({ description: `One of: ${ACTIONS}` }),
|
|
42
42
|
url: Type.Optional(Type.String({ description: "navigate/new_page: URL, or back|forward|reload for navigate" })),
|
|
43
|
-
selector: Type.Optional(Type.String({ description: "CSS selector for click/fill/hover/select/wait_for" })),
|
|
43
|
+
selector: Type.Optional(Type.String({ description: "CSS selector for click/fill/hover/select/wait_for/extract" })),
|
|
44
44
|
text: Type.Optional(Type.String({ description: "fill: text to type; wait_for: text to await" })),
|
|
45
45
|
key: Type.Optional(Type.String({ description: "press: key or combo, e.g. Enter, Control+A" })),
|
|
46
46
|
values: Type.Optional(Type.Array(Type.String(), { description: "select: option values" })),
|
|
@@ -1,15 +1,164 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "opl-browser",
|
|
3
|
-
"version": "1.
|
|
3
|
+
"version": "1.1.0",
|
|
4
4
|
"lockfileVersion": 3,
|
|
5
5
|
"requires": true,
|
|
6
6
|
"packages": {
|
|
7
7
|
"": {
|
|
8
8
|
"name": "opl-browser",
|
|
9
|
-
"version": "1.
|
|
9
|
+
"version": "1.1.0",
|
|
10
10
|
"license": "ISC",
|
|
11
11
|
"dependencies": {
|
|
12
|
-
"
|
|
12
|
+
"@mozilla/readability": "^0.6.0",
|
|
13
|
+
"linkedom": "^0.18.13",
|
|
14
|
+
"playwright": "^1.49.0",
|
|
15
|
+
"turndown": "^7.2.4"
|
|
16
|
+
}
|
|
17
|
+
},
|
|
18
|
+
"node_modules/@mixmark-io/domino": {
|
|
19
|
+
"version": "2.2.0",
|
|
20
|
+
"resolved": "https://registry.npmjs.org/@mixmark-io/domino/-/domino-2.2.0.tgz",
|
|
21
|
+
"integrity": "sha512-Y28PR25bHXUg88kCV7nivXrP2Nj2RueZ3/l/jdx6J9f8J4nsEGcgX0Qe6lt7Pa+J79+kPiJU3LguR6O/6zrLOw==",
|
|
22
|
+
"license": "BSD-2-Clause"
|
|
23
|
+
},
|
|
24
|
+
"node_modules/@mozilla/readability": {
|
|
25
|
+
"version": "0.6.0",
|
|
26
|
+
"resolved": "https://registry.npmjs.org/@mozilla/readability/-/readability-0.6.0.tgz",
|
|
27
|
+
"integrity": "sha512-juG5VWh4qAivzTAeMzvY9xs9HY5rAcr2E4I7tiSSCokRFi7XIZCAu92ZkSTsIj1OPceCifL3cpfteP3pDT9/QQ==",
|
|
28
|
+
"license": "Apache-2.0",
|
|
29
|
+
"engines": {
|
|
30
|
+
"node": ">=14.0.0"
|
|
31
|
+
}
|
|
32
|
+
},
|
|
33
|
+
"node_modules/boolbase": {
|
|
34
|
+
"version": "2.0.0",
|
|
35
|
+
"resolved": "https://registry.npmjs.org/boolbase/-/boolbase-2.0.0.tgz",
|
|
36
|
+
"integrity": "sha512-DkVaaQHymRhpYEYo9x1oo7Q7B0Y6KJUsjm3c9eTyFDby4MHLBTwZ6ZDWBel5zrYxj1WsZgC5oLpiz+93MluXeA==",
|
|
37
|
+
"license": "ISC",
|
|
38
|
+
"engines": {
|
|
39
|
+
"node": ">=20.19.0"
|
|
40
|
+
},
|
|
41
|
+
"funding": {
|
|
42
|
+
"type": "github",
|
|
43
|
+
"url": "https://github.com/sponsors/fb55"
|
|
44
|
+
}
|
|
45
|
+
},
|
|
46
|
+
"node_modules/css-select": {
|
|
47
|
+
"version": "7.0.0",
|
|
48
|
+
"resolved": "https://registry.npmjs.org/css-select/-/css-select-7.0.0.tgz",
|
|
49
|
+
"integrity": "sha512-snmjEVXy+1LnwXdxhYvTMj1d9tOh4HxkA1YmoayVBeeyR2C14Pum7fcxJIm4SswYspVy866eYNwlH6xC3/VH5g==",
|
|
50
|
+
"license": "BSD-2-Clause",
|
|
51
|
+
"dependencies": {
|
|
52
|
+
"boolbase": "^2.0.0",
|
|
53
|
+
"css-what": "^8.0.0",
|
|
54
|
+
"domhandler": "^6.0.1",
|
|
55
|
+
"domutils": "^4.0.2",
|
|
56
|
+
"nth-check": "^3.0.1"
|
|
57
|
+
},
|
|
58
|
+
"engines": {
|
|
59
|
+
"node": ">=20.19.0"
|
|
60
|
+
},
|
|
61
|
+
"funding": {
|
|
62
|
+
"type": "github",
|
|
63
|
+
"url": "https://github.com/sponsors/fb55"
|
|
64
|
+
}
|
|
65
|
+
},
|
|
66
|
+
"node_modules/css-what": {
|
|
67
|
+
"version": "8.0.0",
|
|
68
|
+
"resolved": "https://registry.npmjs.org/css-what/-/css-what-8.0.0.tgz",
|
|
69
|
+
"integrity": "sha512-DH0Bqq3DNp5tdOReuNyAA+Ev4Y2GS5FMbZpeTLP6C4CDi0h5nL0BmUPChXw3o/qbHLDWHl49sbNqQVY7bMSDdw==",
|
|
70
|
+
"license": "BSD-2-Clause",
|
|
71
|
+
"engines": {
|
|
72
|
+
"node": ">=20.19.0"
|
|
73
|
+
},
|
|
74
|
+
"funding": {
|
|
75
|
+
"type": "github",
|
|
76
|
+
"url": "https://github.com/sponsors/fb55"
|
|
77
|
+
}
|
|
78
|
+
},
|
|
79
|
+
"node_modules/cssom": {
|
|
80
|
+
"version": "0.5.0",
|
|
81
|
+
"resolved": "https://registry.npmjs.org/cssom/-/cssom-0.5.0.tgz",
|
|
82
|
+
"integrity": "sha512-iKuQcq+NdHqlAcwUY0o/HL69XQrUaQdMjmStJ8JFmUaiiQErlhrmuigkg/CU4E2J0IyUKUrMAgl36TvN67MqTw==",
|
|
83
|
+
"license": "MIT"
|
|
84
|
+
},
|
|
85
|
+
"node_modules/dom-serializer": {
|
|
86
|
+
"version": "3.1.1",
|
|
87
|
+
"resolved": "https://registry.npmjs.org/dom-serializer/-/dom-serializer-3.1.1.tgz",
|
|
88
|
+
"integrity": "sha512-4MEa38/QexBob6gFNwu+EGdWvhJ1OKuNwdYY3Y3NyeWDQfnGeDYQUDfIRzWu5B5gsv03so2Uxd28YC6zrsx3Lw==",
|
|
89
|
+
"license": "MIT",
|
|
90
|
+
"dependencies": {
|
|
91
|
+
"domelementtype": "^3.0.0",
|
|
92
|
+
"domhandler": "^6.0.0",
|
|
93
|
+
"entities": "^8.0.0"
|
|
94
|
+
},
|
|
95
|
+
"engines": {
|
|
96
|
+
"node": ">=20.19.0"
|
|
97
|
+
},
|
|
98
|
+
"funding": {
|
|
99
|
+
"type": "github",
|
|
100
|
+
"url": "https://github.com/cheeriojs/dom-serializer?sponsor=1"
|
|
101
|
+
}
|
|
102
|
+
},
|
|
103
|
+
"node_modules/domelementtype": {
|
|
104
|
+
"version": "3.0.0",
|
|
105
|
+
"resolved": "https://registry.npmjs.org/domelementtype/-/domelementtype-3.0.0.tgz",
|
|
106
|
+
"integrity": "sha512-umCQid3jKbDmVjx8jGaW7uUykm4DEUeyV21hPxNMo2nV955DhUThwqyOIDtreepP31hl84X7G5U9ZfsWvIB3Pg==",
|
|
107
|
+
"funding": [
|
|
108
|
+
{
|
|
109
|
+
"type": "github",
|
|
110
|
+
"url": "https://github.com/sponsors/fb55"
|
|
111
|
+
}
|
|
112
|
+
],
|
|
113
|
+
"license": "BSD-2-Clause",
|
|
114
|
+
"engines": {
|
|
115
|
+
"node": ">=20.19.0"
|
|
116
|
+
}
|
|
117
|
+
},
|
|
118
|
+
"node_modules/domhandler": {
|
|
119
|
+
"version": "6.0.1",
|
|
120
|
+
"resolved": "https://registry.npmjs.org/domhandler/-/domhandler-6.0.1.tgz",
|
|
121
|
+
"integrity": "sha512-gYzvtM72ZtxQO0T048kd6HWSbbGCNOUwcnfQ01cqIJ4X2IYKFFHZ5mKvrQETcFXxsRObZulDaKmy//R7TPtsBg==",
|
|
122
|
+
"license": "BSD-2-Clause",
|
|
123
|
+
"dependencies": {
|
|
124
|
+
"domelementtype": "^3.0.0"
|
|
125
|
+
},
|
|
126
|
+
"engines": {
|
|
127
|
+
"node": ">=20.19.0"
|
|
128
|
+
},
|
|
129
|
+
"funding": {
|
|
130
|
+
"type": "github",
|
|
131
|
+
"url": "https://github.com/fb55/domhandler?sponsor=1"
|
|
132
|
+
}
|
|
133
|
+
},
|
|
134
|
+
"node_modules/domutils": {
|
|
135
|
+
"version": "4.0.2",
|
|
136
|
+
"resolved": "https://registry.npmjs.org/domutils/-/domutils-4.0.2.tgz",
|
|
137
|
+
"integrity": "sha512-qI4JLRKnSzqFqr7hAlS5xQDusBCjKSEG4t4+7aNrIQMHBcsC2TGEhuyABJdYkgSewL57PNLYEiibY2iPKhKpaA==",
|
|
138
|
+
"license": "BSD-2-Clause",
|
|
139
|
+
"dependencies": {
|
|
140
|
+
"dom-serializer": "^3.0.0",
|
|
141
|
+
"domelementtype": "^3.0.0",
|
|
142
|
+
"domhandler": "^6.0.0"
|
|
143
|
+
},
|
|
144
|
+
"engines": {
|
|
145
|
+
"node": ">=20.19.0"
|
|
146
|
+
},
|
|
147
|
+
"funding": {
|
|
148
|
+
"type": "github",
|
|
149
|
+
"url": "https://github.com/fb55/domutils?sponsor=1"
|
|
150
|
+
}
|
|
151
|
+
},
|
|
152
|
+
"node_modules/entities": {
|
|
153
|
+
"version": "8.1.0",
|
|
154
|
+
"resolved": "https://registry.npmjs.org/entities/-/entities-8.1.0.tgz",
|
|
155
|
+
"integrity": "sha512-kxL7msIffSuh9aaFAMD7rxAIuTRMAHMeBtgHW2yUdWw732ZNh4MehkF2gdjvtdmikkaIP9bFDDJOPlsvm7avrA==",
|
|
156
|
+
"license": "BSD-2-Clause",
|
|
157
|
+
"engines": {
|
|
158
|
+
"node": ">=20.19.0"
|
|
159
|
+
},
|
|
160
|
+
"funding": {
|
|
161
|
+
"url": "https://github.com/fb55/entities?sponsor=1"
|
|
13
162
|
}
|
|
14
163
|
},
|
|
15
164
|
"node_modules/fsevents": {
|
|
@@ -26,6 +175,150 @@
|
|
|
26
175
|
"node": "^8.16.0 || ^10.6.0 || >=11.0.0"
|
|
27
176
|
}
|
|
28
177
|
},
|
|
178
|
+
"node_modules/html-escaper": {
|
|
179
|
+
"version": "3.0.3",
|
|
180
|
+
"resolved": "https://registry.npmjs.org/html-escaper/-/html-escaper-3.0.3.tgz",
|
|
181
|
+
"integrity": "sha512-RuMffC89BOWQoY0WKGpIhn5gX3iI54O6nRA0yC124NYVtzjmFWBIiFd8M0x+ZdX0P9R4lADg1mgP8C7PxGOWuQ==",
|
|
182
|
+
"license": "MIT"
|
|
183
|
+
},
|
|
184
|
+
"node_modules/htmlparser2": {
|
|
185
|
+
"version": "10.1.0",
|
|
186
|
+
"resolved": "https://registry.npmjs.org/htmlparser2/-/htmlparser2-10.1.0.tgz",
|
|
187
|
+
"integrity": "sha512-VTZkM9GWRAtEpveh7MSF6SjjrpNVNNVJfFup7xTY3UpFtm67foy9HDVXneLtFVt4pMz5kZtgNcvCniNFb1hlEQ==",
|
|
188
|
+
"funding": [
|
|
189
|
+
"https://github.com/fb55/htmlparser2?sponsor=1",
|
|
190
|
+
{
|
|
191
|
+
"type": "github",
|
|
192
|
+
"url": "https://github.com/sponsors/fb55"
|
|
193
|
+
}
|
|
194
|
+
],
|
|
195
|
+
"license": "MIT",
|
|
196
|
+
"dependencies": {
|
|
197
|
+
"domelementtype": "^2.3.0",
|
|
198
|
+
"domhandler": "^5.0.3",
|
|
199
|
+
"domutils": "^3.2.2",
|
|
200
|
+
"entities": "^7.0.1"
|
|
201
|
+
}
|
|
202
|
+
},
|
|
203
|
+
"node_modules/htmlparser2/node_modules/dom-serializer": {
|
|
204
|
+
"version": "2.0.0",
|
|
205
|
+
"resolved": "https://registry.npmjs.org/dom-serializer/-/dom-serializer-2.0.0.tgz",
|
|
206
|
+
"integrity": "sha512-wIkAryiqt/nV5EQKqQpo3SToSOV9J0DnbJqwK7Wv/Trc92zIAYZ4FlMu+JPFW1DfGFt81ZTCGgDEabffXeLyJg==",
|
|
207
|
+
"license": "MIT",
|
|
208
|
+
"dependencies": {
|
|
209
|
+
"domelementtype": "^2.3.0",
|
|
210
|
+
"domhandler": "^5.0.2",
|
|
211
|
+
"entities": "^4.2.0"
|
|
212
|
+
},
|
|
213
|
+
"funding": {
|
|
214
|
+
"url": "https://github.com/cheeriojs/dom-serializer?sponsor=1"
|
|
215
|
+
}
|
|
216
|
+
},
|
|
217
|
+
"node_modules/htmlparser2/node_modules/dom-serializer/node_modules/entities": {
|
|
218
|
+
"version": "4.5.0",
|
|
219
|
+
"resolved": "https://registry.npmjs.org/entities/-/entities-4.5.0.tgz",
|
|
220
|
+
"integrity": "sha512-V0hjH4dGPh9Ao5p0MoRY6BVqtwCjhz6vI5LT8AJ55H+4g9/4vbHx1I54fS0XuclLhDHArPQCiMjDxjaL8fPxhw==",
|
|
221
|
+
"license": "BSD-2-Clause",
|
|
222
|
+
"engines": {
|
|
223
|
+
"node": ">=0.12"
|
|
224
|
+
},
|
|
225
|
+
"funding": {
|
|
226
|
+
"url": "https://github.com/fb55/entities?sponsor=1"
|
|
227
|
+
}
|
|
228
|
+
},
|
|
229
|
+
"node_modules/htmlparser2/node_modules/domelementtype": {
|
|
230
|
+
"version": "2.3.0",
|
|
231
|
+
"resolved": "https://registry.npmjs.org/domelementtype/-/domelementtype-2.3.0.tgz",
|
|
232
|
+
"integrity": "sha512-OLETBj6w0OsagBwdXnPdN0cnMfF9opN69co+7ZrbfPGrdpPVNBUj02spi6B1N7wChLQiPn4CSH/zJvXw56gmHw==",
|
|
233
|
+
"funding": [
|
|
234
|
+
{
|
|
235
|
+
"type": "github",
|
|
236
|
+
"url": "https://github.com/sponsors/fb55"
|
|
237
|
+
}
|
|
238
|
+
],
|
|
239
|
+
"license": "BSD-2-Clause"
|
|
240
|
+
},
|
|
241
|
+
"node_modules/htmlparser2/node_modules/domhandler": {
|
|
242
|
+
"version": "5.0.3",
|
|
243
|
+
"resolved": "https://registry.npmjs.org/domhandler/-/domhandler-5.0.3.tgz",
|
|
244
|
+
"integrity": "sha512-cgwlv/1iFQiFnU96XXgROh8xTeetsnJiDsTc7TYCLFd9+/WNkIqPTxiM/8pSd8VIrhXGTf1Ny1q1hquVqDJB5w==",
|
|
245
|
+
"license": "BSD-2-Clause",
|
|
246
|
+
"dependencies": {
|
|
247
|
+
"domelementtype": "^2.3.0"
|
|
248
|
+
},
|
|
249
|
+
"engines": {
|
|
250
|
+
"node": ">= 4"
|
|
251
|
+
},
|
|
252
|
+
"funding": {
|
|
253
|
+
"url": "https://github.com/fb55/domhandler?sponsor=1"
|
|
254
|
+
}
|
|
255
|
+
},
|
|
256
|
+
"node_modules/htmlparser2/node_modules/domutils": {
|
|
257
|
+
"version": "3.2.2",
|
|
258
|
+
"resolved": "https://registry.npmjs.org/domutils/-/domutils-3.2.2.tgz",
|
|
259
|
+
"integrity": "sha512-6kZKyUajlDuqlHKVX1w7gyslj9MPIXzIFiz/rGu35uC1wMi+kMhQwGhl4lt9unC9Vb9INnY9Z3/ZA3+FhASLaw==",
|
|
260
|
+
"license": "BSD-2-Clause",
|
|
261
|
+
"dependencies": {
|
|
262
|
+
"dom-serializer": "^2.0.0",
|
|
263
|
+
"domelementtype": "^2.3.0",
|
|
264
|
+
"domhandler": "^5.0.3"
|
|
265
|
+
},
|
|
266
|
+
"funding": {
|
|
267
|
+
"url": "https://github.com/fb55/domutils?sponsor=1"
|
|
268
|
+
}
|
|
269
|
+
},
|
|
270
|
+
"node_modules/htmlparser2/node_modules/entities": {
|
|
271
|
+
"version": "7.0.1",
|
|
272
|
+
"resolved": "https://registry.npmjs.org/entities/-/entities-7.0.1.tgz",
|
|
273
|
+
"integrity": "sha512-TWrgLOFUQTH994YUyl1yT4uyavY5nNB5muff+RtWaqNVCAK408b5ZnnbNAUEWLTCpum9w6arT70i1XdQ4UeOPA==",
|
|
274
|
+
"license": "BSD-2-Clause",
|
|
275
|
+
"engines": {
|
|
276
|
+
"node": ">=0.12"
|
|
277
|
+
},
|
|
278
|
+
"funding": {
|
|
279
|
+
"url": "https://github.com/fb55/entities?sponsor=1"
|
|
280
|
+
}
|
|
281
|
+
},
|
|
282
|
+
"node_modules/linkedom": {
|
|
283
|
+
"version": "0.18.13",
|
|
284
|
+
"resolved": "https://registry.npmjs.org/linkedom/-/linkedom-0.18.13.tgz",
|
|
285
|
+
"integrity": "sha512-ES/o9qotMpzpN2MHs+Iq/JcVoOj8Fa5wiQYrTdFpvAnwXL0g66XHHUc9WUMk6nAlBtGsFQ24ne+SYnvnaQ2FSw==",
|
|
286
|
+
"license": "ISC",
|
|
287
|
+
"dependencies": {
|
|
288
|
+
"css-select": "^7.0.0",
|
|
289
|
+
"cssom": "^0.5.0",
|
|
290
|
+
"html-escaper": "^3.0.3",
|
|
291
|
+
"htmlparser2": "^10.1.0",
|
|
292
|
+
"uhyphen": "^0.2.0"
|
|
293
|
+
},
|
|
294
|
+
"engines": {
|
|
295
|
+
"node": ">=16"
|
|
296
|
+
},
|
|
297
|
+
"peerDependencies": {
|
|
298
|
+
"canvas": ">= 2"
|
|
299
|
+
},
|
|
300
|
+
"peerDependenciesMeta": {
|
|
301
|
+
"canvas": {
|
|
302
|
+
"optional": true
|
|
303
|
+
}
|
|
304
|
+
}
|
|
305
|
+
},
|
|
306
|
+
"node_modules/nth-check": {
|
|
307
|
+
"version": "3.0.1",
|
|
308
|
+
"resolved": "https://registry.npmjs.org/nth-check/-/nth-check-3.0.1.tgz",
|
|
309
|
+
"integrity": "sha512-GX0gsdbGVCgnRgbeGaubfjpBXyYRWOOCVeYh08bSQvDZqxz5ndXs1OTfAt/h36G1xvI94YIspsI0sVFqAV9+RQ==",
|
|
310
|
+
"license": "BSD-2-Clause",
|
|
311
|
+
"dependencies": {
|
|
312
|
+
"boolbase": "^2.0.0"
|
|
313
|
+
},
|
|
314
|
+
"engines": {
|
|
315
|
+
"node": ">=20.19.0"
|
|
316
|
+
},
|
|
317
|
+
"funding": {
|
|
318
|
+
"type": "github",
|
|
319
|
+
"url": "https://github.com/fb55/nth-check?sponsor=1"
|
|
320
|
+
}
|
|
321
|
+
},
|
|
29
322
|
"node_modules/playwright": {
|
|
30
323
|
"version": "1.62.1",
|
|
31
324
|
"resolved": "https://registry.npmjs.org/playwright/-/playwright-1.62.1.tgz",
|
|
@@ -55,6 +348,25 @@
|
|
|
55
348
|
"engines": {
|
|
56
349
|
"node": ">=20"
|
|
57
350
|
}
|
|
351
|
+
},
|
|
352
|
+
"node_modules/turndown": {
|
|
353
|
+
"version": "7.2.4",
|
|
354
|
+
"resolved": "https://registry.npmjs.org/turndown/-/turndown-7.2.4.tgz",
|
|
355
|
+
"integrity": "sha512-I8yFsfRzmzK0WV1pNNOA4A7y4RDfFxPRxb3t+e3ui14qSGOxGtiSP6GjeX+Y6CHb7HYaFj7ECUD7VE5kQMZWGQ==",
|
|
356
|
+
"license": "MIT",
|
|
357
|
+
"dependencies": {
|
|
358
|
+
"@mixmark-io/domino": "^2.2.0"
|
|
359
|
+
},
|
|
360
|
+
"engines": {
|
|
361
|
+
"node": ">=18",
|
|
362
|
+
"npm": ">=9"
|
|
363
|
+
}
|
|
364
|
+
},
|
|
365
|
+
"node_modules/uhyphen": {
|
|
366
|
+
"version": "0.2.0",
|
|
367
|
+
"resolved": "https://registry.npmjs.org/uhyphen/-/uhyphen-0.2.0.tgz",
|
|
368
|
+
"integrity": "sha512-qz3o9CHXmJJPGBdqzab7qAYuW8kQGKNEuoHFYrBwV6hWIMcpAmxDLXojcHfFr9US1Pe6zUswEIJIbLI610fuqA==",
|
|
369
|
+
"license": "ISC"
|
|
58
370
|
}
|
|
59
371
|
}
|
|
60
372
|
}
|
|
@@ -1,10 +1,13 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "opl-browser",
|
|
3
|
-
"version": "1.
|
|
3
|
+
"version": "1.1.0",
|
|
4
4
|
"description": "Chromium browser automation for pi via Playwright. A single dispatcher tool for navigation, snapshots, interaction, console/network capture, and evaluation — replaces the chrome-devtools MCP server.",
|
|
5
5
|
"main": "index.js",
|
|
6
6
|
"license": "ISC",
|
|
7
7
|
"dependencies": {
|
|
8
|
-
"
|
|
8
|
+
"@mozilla/readability": "^0.6.0",
|
|
9
|
+
"linkedom": "^0.18.13",
|
|
10
|
+
"playwright": "^1.49.0",
|
|
11
|
+
"turndown": "^7.2.4"
|
|
9
12
|
}
|
|
10
13
|
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
declare module "turndown";
|
|
@@ -15,7 +15,7 @@ Auditable Pi model benchmark for closed-answer contracts, JSON instruction follo
|
|
|
15
15
|
/simplebench <model> --llama-server
|
|
16
16
|
/simplebench <model> --llamagputop
|
|
17
17
|
/simplebench <model> --tag=<word>
|
|
18
|
-
/simplebench <model> --sequence
|
|
18
|
+
/simplebench <model> --sequence[=<name>]
|
|
19
19
|
/simplebench --all [--no-artifact]
|
|
20
20
|
/simplebench --all --test-all
|
|
21
21
|
/simplebench --help
|
|
@@ -42,7 +42,7 @@ simplebench({ model: "global.openai.gpt-5.6-terra", test_all: true })
|
|
|
42
42
|
- Six opt-in execution-backed coding tasks in disposable directories (`--coding-lite`).
|
|
43
43
|
- `--test-all` runs baseline, coding-lite, and deterministic grounded research; `--research-live` adds the configured live research-artifact integration smoke test. `--all` still selects every Ollama model.
|
|
44
44
|
- `--3ptest` is the explicit form of the default baseline suite (reasoning, instruction following, tool calls). It changes nothing about the run; artifacts already name the baseline suite `3ptest`.
|
|
45
|
-
- `--sequence` runs
|
|
45
|
+
- `--sequence[=<name>]` runs a named templated multi-iteration profile stored in `opl-simplebench.json` (`runSequence.sequences`): sequential iterations, per-entry flags and `--tag`, `llamaMetrics` and `pauseMs` cooldown with block-level defaults a profile can override. See "Run a templated sequence" below.
|
|
46
46
|
- `--llama-server` (`llama_server: true`) captures configured direct llama-server `/props` and `/metrics` without changing inference routing, so it works with a LiteLLM proxy.
|
|
47
47
|
- `--llamagputop` (`llamagputop: true`) captures configured llama.cpp `/stats` metadata independently of the inference provider; its served model ID is authoritative.
|
|
48
48
|
- `--tag=<word>` (`tag: "<word>"`) labels a run: stored as `benchmark.tag` and prefixed onto the artifact file or bundle name, for example `simplebench-coldrun-coding-lite-<model>-...`. The tag must be a single word (letters, digits, dot, dash, underscore).
|
|
@@ -87,10 +87,10 @@ Use this in sensitive environments. The terminal summary still reports category
|
|
|
87
87
|
### Run a templated sequence
|
|
88
88
|
|
|
89
89
|
```text
|
|
90
|
-
/simplebench --sequence
|
|
90
|
+
/simplebench --sequence=<name>
|
|
91
91
|
```
|
|
92
92
|
|
|
93
|
-
Repeating a warm-up protocol (first-run-after-boot coding check, then baseline, then the full suite) means re-typing the same flags every session. `--sequence` reads
|
|
93
|
+
Repeating a warm-up protocol (first-run-after-boot coding check, then baseline, then the full suite) means re-typing the same flags every session. `--sequence` reads a named `runSequence` profile from `~/.pi/agent/configs/opl-simplebench.json` and runs each entry as its own iteration against the current model (or the model passed with `--sequence`):
|
|
94
94
|
|
|
95
95
|
```json
|
|
96
96
|
{
|
|
@@ -98,22 +98,36 @@ Repeating a warm-up protocol (first-run-after-boot coding check, then baseline,
|
|
|
98
98
|
"enabled": true,
|
|
99
99
|
"llamaMetrics": true,
|
|
100
100
|
"pauseMs": 5000,
|
|
101
|
-
"
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
101
|
+
"sequences": [
|
|
102
|
+
{
|
|
103
|
+
"name": "warmup",
|
|
104
|
+
"iterations": [
|
|
105
|
+
"--coding-lite --tag=coldest",
|
|
106
|
+
"--3ptest --tag=colder",
|
|
107
|
+
"--coding-lite --tag=semiwarm",
|
|
108
|
+
"--test-all --research-live --tag=warm"
|
|
109
|
+
]
|
|
110
|
+
},
|
|
111
|
+
{
|
|
112
|
+
"name": "coding-only",
|
|
113
|
+
"llamaMetrics": false,
|
|
114
|
+
"pauseMs": 3000,
|
|
115
|
+
"iterations": ["--coding-lite --tag=coldest", "--coding-lite --tag=warmer"]
|
|
116
|
+
}
|
|
106
117
|
]
|
|
107
118
|
}
|
|
108
119
|
}
|
|
109
120
|
```
|
|
110
121
|
|
|
111
122
|
- `enabled` gates the flag: `/simplebench --sequence` errors when it is missing or `false`. A bare `/simplebench` is unaffected.
|
|
112
|
-
-
|
|
123
|
+
- Profile names are single words (letters, digits, dot, dash, underscore) and must be unique. `/simplebench --sequence` without a name runs the only defined profile; with several profiles it errors and lists the available names.
|
|
124
|
+
- Block-level `llamaMetrics` and `pauseMs` are defaults; a profile may override either, so a coding-only curve can skip server stats or use a different cooldown.
|
|
125
|
+
- Each `iterations` entry is the same flag string you would type after `/simplebench`, including its own `--tag`.
|
|
113
126
|
- `llamaMetrics: true` appends `--llama-server --llamagputop` to every entry, so local server stats are captured per iteration without repeating the flags.
|
|
114
127
|
- `pauseMs` sleeps between iterations (never after the last one), including after a failed iteration, so thermal/KV-cache cooldown stays consistent across the warm-up curve.
|
|
115
128
|
- Iterations run sequentially and each writes its normal artifact/bundle; a failed iteration is reported and the sequence continues. A final summary line counts completed iterations.
|
|
116
|
-
- Entries containing `--all` or `--sequence` are rejected before anything runs, as are invalid configurations (
|
|
129
|
+
- Entries containing `--all` or `--sequence` are rejected before anything runs, as are invalid configurations (no profiles, empty `iterations`, negative `pauseMs`) and invalid per-entry tags. The outer `--tag` is ignored; each entry carries its own.
|
|
130
|
+
- The legacy flat `runSequence.sequence` array still works as an anonymous profile, so existing configs need no change. Selecting it by name is not possible; migrate to `sequences` when you need more than one protocol.
|
|
117
131
|
|
|
118
132
|
### Grounded research in `--test-all`
|
|
119
133
|
|
|
@@ -138,6 +138,7 @@ function makeOpenAiChatFn(baseUrl: string, apiKey?: string, thinkingMax = false)
|
|
|
138
138
|
|
|
139
139
|
const controller = new AbortController();
|
|
140
140
|
const timeoutId = setTimeout(() => controller.abort(), CONFIG.TOOL_TEST_TIMEOUT_MS);
|
|
141
|
+
const startedAt = new Date().toISOString();
|
|
141
142
|
const start = Date.now();
|
|
142
143
|
try {
|
|
143
144
|
const res = await fetch(`${baseUrl}/chat/completions`, {
|
|
@@ -163,6 +164,8 @@ function makeOpenAiChatFn(baseUrl: string, apiKey?: string, thinkingMax = false)
|
|
|
163
164
|
toolCalls: toolCalls && toolCalls.length > 0 ? toolCalls : undefined,
|
|
164
165
|
elapsedMs,
|
|
165
166
|
raw: parsed,
|
|
167
|
+
startedAt,
|
|
168
|
+
finishedAt: new Date().toISOString(),
|
|
166
169
|
};
|
|
167
170
|
} catch (e: any) {
|
|
168
171
|
clearTimeout(timeoutId);
|
|
@@ -265,6 +268,7 @@ function makeBedrockChatFn(providerInfo: { region?: string }): ChatFn {
|
|
|
265
268
|
|
|
266
269
|
const controller = new AbortController();
|
|
267
270
|
const timeoutId = setTimeout(() => controller.abort(), CONFIG.TOOL_TEST_TIMEOUT_MS);
|
|
271
|
+
const startedAt = new Date().toISOString();
|
|
268
272
|
const start = Date.now();
|
|
269
273
|
try {
|
|
270
274
|
const res = await fetch(`https://${host}${urlPath}`, {
|
|
@@ -284,7 +288,7 @@ function makeBedrockChatFn(providerInfo: { region?: string }): ChatFn {
|
|
|
284
288
|
const toolCalls = blocks.filter((b: any) => b.toolUse).map((b: any) => ({
|
|
285
289
|
function: { name: b.toolUse.name, arguments: b.toolUse.input ?? {} },
|
|
286
290
|
}));
|
|
287
|
-
return { content, toolCalls: toolCalls.length > 0 ? toolCalls : undefined, elapsedMs, raw: parsed };
|
|
291
|
+
return { content, toolCalls: toolCalls.length > 0 ? toolCalls : undefined, elapsedMs, raw: parsed, startedAt, finishedAt: new Date().toISOString() };
|
|
288
292
|
} catch (e: any) {
|
|
289
293
|
clearTimeout(timeoutId);
|
|
290
294
|
if (e.name === "AbortError") throw new Error(`Bedrock API timed out after ${msHuman(CONFIG.TOOL_TEST_TIMEOUT_MS)}`);
|
|
@@ -344,6 +348,7 @@ function makeOllamaToolChatFn(): ChatFn {
|
|
|
344
348
|
|
|
345
349
|
const controller = new AbortController();
|
|
346
350
|
const timeoutId = setTimeout(() => controller.abort(), CONFIG.TOOL_TEST_TIMEOUT_MS);
|
|
351
|
+
const startedAt = new Date().toISOString();
|
|
347
352
|
const start = Date.now();
|
|
348
353
|
try {
|
|
349
354
|
const res = await fetch(`${ollamaBase()}/api/chat`, {
|
|
@@ -370,6 +375,8 @@ function makeOllamaToolChatFn(): ChatFn {
|
|
|
370
375
|
toolCalls: toolCalls && toolCalls.length > 0 ? toolCalls : undefined,
|
|
371
376
|
elapsedMs,
|
|
372
377
|
raw: parsed,
|
|
378
|
+
startedAt,
|
|
379
|
+
finishedAt: new Date().toISOString(),
|
|
373
380
|
};
|
|
374
381
|
} catch (e: any) {
|
|
375
382
|
clearTimeout(timeoutId);
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import type { ExtensionAPI, AgentToolResult } from "@earendil-works/pi-coding-agent";
|
|
2
2
|
import { debugLog } from "./util/debug";
|
|
3
3
|
import { detectProvider } from "./util/providers";
|
|
4
|
-
import { readTestConfig, TOOL_SUPPORT_CACHE_PATH, type ModelTestUserConfig } from "./util/config";
|
|
4
|
+
import { readTestConfig, TOOL_SUPPORT_CACHE_PATH, type ModelTestUserConfig, type RunSequenceProfile } from "./util/config";
|
|
5
5
|
import type { SimplebenchOptions } from "./types";
|
|
6
6
|
import { createBenchmark } from "./benchmark";
|
|
7
7
|
|
|
@@ -13,25 +13,50 @@ export function parseCommandArgs(args: string): SimplebenchOptions {
|
|
|
13
13
|
tag = tagToken.slice("--tag=".length);
|
|
14
14
|
if (!/^[A-Za-z0-9][A-Za-z0-9._-]*$/.test(tag)) throw new Error(`--tag must be a single word (letters, digits, dot, dash, underscore): got "${tag}"`);
|
|
15
15
|
}
|
|
16
|
-
|
|
16
|
+
const seqToken = tokens.find(token => token === "--sequence" || token.startsWith("--sequence="));
|
|
17
|
+
let sequence: boolean | string | undefined;
|
|
18
|
+
if (seqToken) {
|
|
19
|
+
if (seqToken === "--sequence") sequence = true;
|
|
20
|
+
else {
|
|
21
|
+
sequence = seqToken.slice("--sequence=".length);
|
|
22
|
+
if (!/^[A-Za-z0-9][A-Za-z0-9._-]*$/.test(sequence)) throw new Error(`--sequence= must name a single word (letters, digits, dot, dash, underscore): got "${seqToken.slice("--sequence=".length)}"`);
|
|
23
|
+
}
|
|
24
|
+
}
|
|
25
|
+
return { model: tokens.find(token => !token.startsWith("--")), allModels: tokens.includes("--all"), writeArtifact: !tokens.includes("--no-artifact"), thinkingMax: tokens.includes("--thinking-max"), codingLite: tokens.includes("--coding-lite"), testAll: tokens.includes("--test-all"), researchLive: tokens.includes("--research-live"), llamaServer: tokens.includes("--llama-server"), llamagputop: tokens.includes("--llamagputop"), ...(tag ? { tag } : {}), ...(tokens.includes("--3ptest") ? { threePTest: true } : {}), ...(sequence ? { sequence } : {}) };
|
|
17
26
|
}
|
|
18
27
|
|
|
19
|
-
export interface ResolvedRunSequence { runs: Array<{ entry: string; options: SimplebenchOptions }>; pauseMs: number }
|
|
28
|
+
export interface ResolvedRunSequence { profile: string; runs: Array<{ entry: string; options: SimplebenchOptions }>; pauseMs: number; llamaMetrics: boolean }
|
|
20
29
|
|
|
21
|
-
|
|
22
|
-
|
|
30
|
+
const PROFILE_NAME_RE = /^[A-Za-z0-9][A-Za-z0-9._-]*$/;
|
|
31
|
+
|
|
32
|
+
/** Expand one configured runSequence profile into per-iteration benchmark options. Throws on disabled/invalid configuration. */
|
|
33
|
+
export function resolveRunSequence(userConfig: ModelTestUserConfig, name?: string): ResolvedRunSequence {
|
|
23
34
|
const rs = userConfig.runSequence;
|
|
24
35
|
if (!rs?.enabled) throw new Error("runSequence is not enabled in opl-simplebench.json");
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
36
|
+
const profiles: RunSequenceProfile[] = [];
|
|
37
|
+
if (Array.isArray(rs.sequence) && rs.sequence.length > 0) profiles.push({ name: "", iterations: rs.sequence });
|
|
38
|
+
for (const profile of rs.sequences ?? []) {
|
|
39
|
+
if (typeof profile?.name !== "string" || !PROFILE_NAME_RE.test(profile.name)) throw new Error(`runSequence profile name must be a single word (letters, digits, dot, dash, underscore): got ${JSON.stringify(profile?.name)}`);
|
|
40
|
+
if (!Array.isArray(profile.iterations) || profile.iterations.length === 0) throw new Error(`runSequence profile "${profile.name}" needs a non-empty iterations array`);
|
|
41
|
+
if (profiles.some(known => known.name === profile.name)) throw new Error(`duplicate runSequence profile name: "${profile.name}"`);
|
|
42
|
+
profiles.push(profile);
|
|
43
|
+
}
|
|
44
|
+
const label = (p: RunSequenceProfile) => p.name || "(legacy)";
|
|
45
|
+
if (profiles.length === 0) throw new Error("runSequence needs a non-empty sequence array or sequences list");
|
|
46
|
+
let chosen: RunSequenceProfile | undefined;
|
|
47
|
+
if (name) chosen = profiles.find(p => p.name === name);
|
|
48
|
+
else if (profiles.length === 1) chosen = profiles[0];
|
|
49
|
+
if (!chosen) throw new Error(name ? `unknown runSequence profile "${name}"; available: ${profiles.map(label).join(", ")}` : `--sequence needs a profile name; available: ${profiles.map(label).join(", ")}`);
|
|
50
|
+
const pauseMs = chosen.pauseMs ?? rs.pauseMs ?? 0;
|
|
51
|
+
if (typeof pauseMs !== "number" || !Number.isFinite(pauseMs) || pauseMs < 0) throw new Error(`runSequence.pauseMs must be a non-negative number: got ${JSON.stringify(chosen.pauseMs ?? rs.pauseMs)}`);
|
|
52
|
+
const llamaMetrics = (chosen.llamaMetrics ?? rs.llamaMetrics) === true;
|
|
53
|
+
const runs = chosen.iterations.map((entry: string) => {
|
|
29
54
|
const tokens = String(entry).trim().split(/\s+/);
|
|
30
55
|
if (tokens.includes("--all")) throw new Error(`runSequence entry cannot contain --all: "${entry}"`);
|
|
31
|
-
if (tokens.
|
|
32
|
-
return { entry: String(entry), options: parseCommandArgs(`${entry}${
|
|
56
|
+
if (tokens.some(token => token === "--sequence" || token.startsWith("--sequence="))) throw new Error(`runSequence entry cannot contain --sequence: "${entry}"`);
|
|
57
|
+
return { entry: String(entry), options: parseCommandArgs(`${entry}${llamaMetrics ? " --llama-server --llamagputop" : ""}`) };
|
|
33
58
|
});
|
|
34
|
-
return { runs, pauseMs };
|
|
59
|
+
return { profile: chosen.name, runs, pauseMs, llamaMetrics };
|
|
35
60
|
}
|
|
36
61
|
|
|
37
62
|
export default function (pi: ExtensionAPI) {
|
|
@@ -40,7 +65,7 @@ export default function (pi: ExtensionAPI) {
|
|
|
40
65
|
|
|
41
66
|
pi.registerCommand("simplebench", {
|
|
42
67
|
description: "Benchmark a model with auditable closed-answer, instruction, and tool-use tests.",
|
|
43
|
-
detailedHelp: "\n\n🔍 Simplebench Extension\n\nThis extension tests AI models across multiple dimensions:\n• Closed-answer contract: 20 deterministic final-line answers (logic, math, spatial, commonsense)\n• Tool Usage: Ability to use available tools effectively\n• Instruction Following: How well the model follows complex JSON instructions\n• Coding Lite: Six isolated, execution-backed coding tasks\n\n📋 Usage Examples:\n /simplebench # Test current model\n /simplebench <model> # Test a specific model\n /simplebench --all # Test all Ollama models\n /simplebench <model> --3ptest # Run the default 3ptest baseline explicitly\n /simplebench <model> --coding-lite # Run only coding tasks\n /simplebench <model> --test-all # Run baseline, coding, and grounded research\n /simplebench <model> --research-live # Add live search integration smoke test\n /simplebench --all --test-all # Run complete suite for every Ollama model\n /simplebench <model> --sequence # Run
|
|
68
|
+
detailedHelp: "\n\n🔍 Simplebench Extension\n\nThis extension tests AI models across multiple dimensions:\n• Closed-answer contract: 20 deterministic final-line answers (logic, math, spatial, commonsense)\n• Tool Usage: Ability to use available tools effectively\n• Instruction Following: How well the model follows complex JSON instructions\n• Coding Lite: Six isolated, execution-backed coding tasks\n\n📋 Usage Examples:\n /simplebench # Test current model\n /simplebench <model> # Test a specific model\n /simplebench --all # Test all Ollama models\n /simplebench <model> --3ptest # Run the default 3ptest baseline explicitly\n /simplebench <model> --coding-lite # Run only coding tasks\n /simplebench <model> --test-all # Run baseline, coding, and grounded research\n /simplebench <model> --research-live # Add live search integration smoke test\n /simplebench --all --test-all # Run complete suite for every Ollama model\n /simplebench <model> --sequence[=<name>] # Run a configured runSequence profile\n /simplebench <model> --thinking-max # Request max reasoning\n /simplebench <model> --llama-server # Capture configured /props and /metrics\n /simplebench <model> --llamagputop # Capture configured llama.cpp stats\n /simplebench <model> --tag=coldrun # Label the run: benchmark.tag plus artifact filename prefix\n /simplebench --help # Show this help\n /simplebench --clear-cache # Clear tool support cache\n\nCoding tasks run in disposable directories and never access the user repository.\n",
|
|
44
69
|
getArgumentCompletions: async (prefix) => {
|
|
45
70
|
try {
|
|
46
71
|
const models = await getOllamaModels();
|
|
@@ -73,7 +98,7 @@ pi.registerCommand("simplebench", {
|
|
|
73
98
|
" /simplebench [model] --research-live - Add live-search integration smoke test\n" +
|
|
74
99
|
" /simplebench [model] --tag=<word> - Label the run (single word; added to benchmark.tag and the artifact name)\n" +
|
|
75
100
|
" /simplebench [model] --3ptest - Run the default baseline suite explicitly\n" +
|
|
76
|
-
" /simplebench [model] --sequence - Run
|
|
101
|
+
" /simplebench [model] --sequence[=<name>] - Run a templated multi-iteration sequence profile from opl-simplebench.json (runSequence.sequences; bare --sequence needs exactly one profile; per-entry --tag, llamaMetrics and pauseMs supported, profile values override the block ones; the outer --tag is ignored)\n" +
|
|
77
102
|
" /simplebench --all --test-all - Run complete suite for all Ollama models\n" +
|
|
78
103
|
" /simplebench --clear-cache - Clear tool support cache\n",
|
|
79
104
|
"info"
|
|
@@ -97,9 +122,10 @@ pi.registerCommand("simplebench", {
|
|
|
97
122
|
}
|
|
98
123
|
|
|
99
124
|
if (parsedArgs.sequence) {
|
|
125
|
+
const requested = typeof parsedArgs.sequence === "string" ? parsedArgs.sequence : undefined;
|
|
100
126
|
let resolved: ResolvedRunSequence;
|
|
101
127
|
try {
|
|
102
|
-
resolved = resolveRunSequence(readTestConfig());
|
|
128
|
+
resolved = resolveRunSequence(readTestConfig(), requested);
|
|
103
129
|
} catch (e: any) {
|
|
104
130
|
ctx.ui.notify(e?.message || String(e), "error");
|
|
105
131
|
return;
|
|
@@ -109,7 +135,7 @@ pi.registerCommand("simplebench", {
|
|
|
109
135
|
ctx.ui.notify("No model specified and no model currently selected", "error");
|
|
110
136
|
return;
|
|
111
137
|
}
|
|
112
|
-
ctx.ui.notify(`Running ${resolved.runs.length}-iteration sequence (${resolved.runs.map(r => r.entry).join(" | ")}
|
|
138
|
+
ctx.ui.notify(`Running ${resolved.runs.length}-iteration sequence${resolved.profile ? ` "${resolved.profile}"` : ""} (llamaMetrics ${resolved.llamaMetrics ? "on" : "off"}, pause ${resolved.pauseMs / 1000}s): ${resolved.runs.map(r => r.entry).join(" | ")}...`, "info");
|
|
113
139
|
let completed = 0;
|
|
114
140
|
for (let i = 0; i < resolved.runs.length; i++) {
|
|
115
141
|
if (i > 0 && resolved.pauseMs > 0) {
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import type { ServerStats } from "./llama-server";
|
|
2
2
|
|
|
3
|
-
export interface SimplebenchOptions { model?: string; allModels: boolean; writeArtifact: boolean; thinkingMax?: boolean; codingLite?: boolean; testAll?: boolean; researchLive?: boolean; llamaServer?: boolean; llamagputop?: boolean; tag?: string; threePTest?: boolean; sequence?: boolean; }
|
|
3
|
+
export interface SimplebenchOptions { model?: string; allModels: boolean; writeArtifact: boolean; thinkingMax?: boolean; codingLite?: boolean; testAll?: boolean; researchLive?: boolean; llamaServer?: boolean; llamagputop?: boolean; tag?: string; threePTest?: boolean; sequence?: boolean | string; }
|
|
4
4
|
export interface RequestMetrics { requestCount: number; retryCount: number; wallTimeMs: number; timeToAnswerMs: number; timeToFirstTokenMs: number | null; startedAt: string | null; finishedAt: string | null; inputTokens: number | null; outputTokens: number | null; totalTokens: number | null; outputTokensPerSecond: number | null; toolCalls: Array<{ name: string; arguments?: unknown }>; }
|
|
5
5
|
export interface TestRecord { id: string; kind: "reasoning" | "instructions" | "tools" | "coding" | "research-grounded" | "research-live"; category?: string; prompt: string; response: string | null; expectedAnswer?: string | string[]; extractedAnswer?: string; score: string; passed: boolean; error: string | null; metrics: RequestMetrics; coding?: { publicPassed: boolean; hiddenPassed: boolean; verifiedAfterEdit: boolean; unrelatedFiles: string[]; toolCalls: number; turns: number; efficiency?: "STRONG" | "MODERATE" | "WEAK" | "FAIL"; }; research?: { toolCalls: string[]; files: string[]; }; }
|
|
6
6
|
export interface RunArtifact { schemaVersion: 1; benchmark: { name: "opl-simplebench"; suite?: "baseline" | "coding-lite" | "test-all"; tag?: string; model: string; provider: string; providerKind: string; thinking: { requested: "default" | "max"; effective: "provider-default" | "openai-reasoning-effort" | "pi-bedrock-reasoning"; level: "max" | null; modelMetadataSource: "active-context" | "scoped-model" | null; }; startedAt: string; finishedAt: string; wallTimeMs: number; artifactEnabled: boolean; }; tests: TestRecord[]; summary: Record<string, unknown> & { serverStats?: ServerStats }; }
|
|
@@ -87,7 +87,18 @@ export interface RunSequenceConfig {
|
|
|
87
87
|
enabled?: boolean;
|
|
88
88
|
llamaMetrics?: boolean;
|
|
89
89
|
pauseMs?: number;
|
|
90
|
+
/** Legacy anonymous sequence; still resolved when no `sequences` profile matches. */
|
|
90
91
|
sequence?: string[];
|
|
92
|
+
/** Named sequences selectable with `/simplebench --sequence=<name>`. */
|
|
93
|
+
sequences?: RunSequenceProfile[];
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
/** One named sequence profile. `llamaMetrics` and `pauseMs` override the block-level values. */
|
|
97
|
+
export interface RunSequenceProfile {
|
|
98
|
+
name: string;
|
|
99
|
+
iterations: string[];
|
|
100
|
+
llamaMetrics?: boolean;
|
|
101
|
+
pauseMs?: number;
|
|
91
102
|
}
|
|
92
103
|
|
|
93
104
|
/**
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@openlines/opl-pi-sht",
|
|
3
|
-
"version": "0.1.
|
|
3
|
+
"version": "0.1.18",
|
|
4
4
|
"description": "Pi coding-agent extension kit: token-efficient context, safe modes, fingerprinted /init, model benchmarking, web access, and browser automation.",
|
|
5
5
|
"repository": {
|
|
6
6
|
"type": "git",
|
|
@@ -41,7 +41,7 @@
|
|
|
41
41
|
"type": "module",
|
|
42
42
|
"scripts": {
|
|
43
43
|
"test": "npm run test:opl-browser && npm run test:opl-footer && npm run test:opl-init && npm run test:opl-input && npm run test:opl-modes && npm run test:opl-questionnaire && npm run test:opl-todo && npm run test:opl-webaccess && npm run test:opl-simplebench && npm run test:opl-ctxtrim",
|
|
44
|
-
"test:opl-browser": "OPL_EXTENSION=opl-browser bun test tests/extension-smoke.test.mjs",
|
|
44
|
+
"test:opl-browser": "bun test tests/opl-browser-helpers.test.mjs && OPL_EXTENSION=opl-browser bun test tests/extension-smoke.test.mjs",
|
|
45
45
|
"test:opl-footer": "bun test tests/opl-footer-helpers.test.mjs && OPL_EXTENSION=opl-footer bun test tests/extension-smoke.test.mjs",
|
|
46
46
|
"test:opl-init": "bun test tests/opl-init-crawl.test.mjs && OPL_EXTENSION=opl-init bun test tests/extension-smoke.test.mjs",
|
|
47
47
|
"test:opl-input": "bun test tests/opl-input-style.test.mjs && OPL_EXTENSION=opl-input bun test tests/extension-smoke.test.mjs",
|