@yahyahammad/venastine 0.3.11 → 0.3.13
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CONFIG_ARCHITECTURE.md +40 -0
- package/README.md +84 -14
- package/cli_commands.py +18 -0
- package/config.yaml +74 -0
- package/config_edit.py +38 -1
- package/config_schema.py +53 -1
- package/core/commands.py +4 -0
- package/core/decision_classifier.py +775 -0
- package/package.json +19 -2
- package/pyproject.toml +15 -1
- package/requirements.txt +8 -8
- package/security/capability.py +7 -1
- package/security/posture.py +85 -1
- package/security/sandbox.py +24 -0
- package/tools/builtin/shell.py +139 -0
- package/tools/builtin/shell_sessions.py +9 -0
- package/tui/app.py +34 -0
package/CONFIG_ARCHITECTURE.md
CHANGED
|
@@ -1228,6 +1228,10 @@ ROADMAP_v2 §28 (G3). WHICH shell commands need a human to say yes.
|
|
|
1228
1228
|
how a command is measured against it. Since §48 (CE1) only
|
|
1229
1229
|
a read-only INERT command runs unasked: anything that may
|
|
1230
1230
|
run code asks, contained or not.
|
|
1231
|
+
"classified" ROADMAP_v3 §56 (JV11): "tiered", plus a decision model
|
|
1232
|
+
asked about a single command that would run in the
|
|
1233
|
+
container and that "tiered" would ask about. A verdict
|
|
1234
|
+
inside every threshold runs it unasked.
|
|
1231
1235
|
"contained" §48 (CE2): the rule "tiered" had before §48. A command the
|
|
1232
1236
|
container confines runs unasked unless it gets network,
|
|
1233
1237
|
code included; reads outside the workspace and network
|
|
@@ -1265,6 +1269,42 @@ security/capability.APPROVAL_MODES, which validate_mode() reads. A second
|
|
|
1265
1269
|
copy here was read by nothing but a test asserting its literal, so the two
|
|
1266
1270
|
could have disagreed and only the unenforced one would have been wrong.
|
|
1267
1271
|
|
|
1272
|
+
### `shell_classifier_style`, `shell_classifier_url`, `shell_classifier_model`, `shell_classifier_credential`
|
|
1273
|
+
|
|
1274
|
+
ROADMAP_v3 §56 (JV7, JV8, JV17). Who the `classified` mode asks, and how.
|
|
1275
|
+
`style` is named and never probed: a probe is a network call at launch and
|
|
1276
|
+
a guess about a security control, and it guesses wrong exactly when a proxy
|
|
1277
|
+
answers 200 to everything. `url` is the base, ending at `/v1`; empty is
|
|
1278
|
+
"not configured" (JV20), and anything that is neither empty nor http(s)
|
|
1279
|
+
fails at launch, because a typo in this endpoint must not look like an
|
|
1280
|
+
outage. `credential` NAMES a `providers.json` entry: the endpoint lives
|
|
1281
|
+
here and the key there, as JV8 split them, and the key is read when a
|
|
1282
|
+
request is made and never held. A key is never sent over plain http to
|
|
1283
|
+
anything but this machine.
|
|
1284
|
+
|
|
1285
|
+
All four are authority keys, and the reason is different for each: the URL
|
|
1286
|
+
decides who answers, the model decides what answers, the credential decides
|
|
1287
|
+
which secret is sent to that URL, and the style decides how the answer is
|
|
1288
|
+
read.
|
|
1289
|
+
|
|
1290
|
+
### `shell_classifier_timeout_s`, `shell_classifier_on_fail`
|
|
1291
|
+
|
|
1292
|
+
ROADMAP_v3 §56 (JV10, JV13, JV18). How long a command waits for a verdict,
|
|
1293
|
+
and what one outside a threshold does. No verdict in time ASKS, under both
|
|
1294
|
+
`on_fail` values -- the one property the section rests on is that making
|
|
1295
|
+
the endpoint unreachable never widens what runs. `ask` ships because the
|
|
1296
|
+
mode has to be usable as a filter before anyone trusts it as a decision.
|
|
1297
|
+
|
|
1298
|
+
### `shell_classifier_min_containedness`, `shell_classifier_max_risk`, `shell_classifier_max_modification`, `shell_classifier_max_network`
|
|
1299
|
+
|
|
1300
|
+
ROADMAP_v3 §56 (JV5, JV16). One threshold per metric, each on 0-1. A
|
|
1301
|
+
`score` answer is the weighted mean of zero-based level indices, divided by
|
|
1302
|
+
levels-1 here so the four share a scale; networkedness is a probability.
|
|
1303
|
+
Containedness passes at or ABOVE its number; the other three at or below.
|
|
1304
|
+
0.75 / 0.25 / 0.5 / 0.5 are starting points, not calibrated values: a
|
|
1305
|
+
threshold tuned against one model is a different policy against another,
|
|
1306
|
+
and a vendor alias (`jev-latest`) moves.
|
|
1307
|
+
|
|
1268
1308
|
### `sandbox_docker_image`
|
|
1269
1309
|
|
|
1270
1310
|
The image both runtimes run: Docker, or Podman when Docker cannot run a
|
package/README.md
CHANGED
|
@@ -1,7 +1,22 @@
|
|
|
1
|
-

|
|
2
|
-
|
|
3
1
|
# Venastine Research Harness
|
|
4
2
|
|
|
3
|
+
**Venastine Research Harness is an open-source, security-first agentic harness for large language models: a tool-using chat assistant and a ten-pass deep-research pipeline that fact-checks, critiques and scores its own claims, in your terminal or in the browser.**
|
|
4
|
+
|
|
5
|
+
<p align="center">
|
|
6
|
+
<a href="https://huggingface.co/spaces/SolenopsisCampo/VenastineHarnessDemo"><img alt="Try the demo on a Hugging Face Space" src="https://img.shields.io/badge/Demo-Hugging%20Face%20Space-1783FF?logo=huggingface&logoColor=white"></a>
|
|
7
|
+
<a href="https://www.modelscope.ai/studios/Venastine-Research/VenastineHarnessDemo"><img alt="Try the demo on a ModelScope Studio" src="https://img.shields.io/badge/Demo-ModelScope%20Studio-1783FF?logo=modelscope&logoColor=white"></a>
|
|
8
|
+
<a href="https://huggingface.co/Venastine-Research"><img alt="Venastine-Research on Hugging Face" src="https://img.shields.io/badge/Hugging%20Face-Venastine--Research-FFC107?logo=huggingface&logoColor=white"></a>
|
|
9
|
+
<a href="https://www.modelscope.ai/organization/Venastine-Research"><img alt="Venastine-Research on ModelScope" src="https://img.shields.io/badge/ModelScope-Venastine--Research-624AFF?logo=modelscope&logoColor=white"></a>
|
|
10
|
+
<br>
|
|
11
|
+
<a href="https://github.com/YahyaHammad/Venastine-Research-Harness/releases/latest"><img alt="Latest GitHub release" src="https://img.shields.io/github/v/release/YahyaHammad/Venastine-Research-Harness?logo=github&label=release"></a>
|
|
12
|
+
<a href="https://www.npmjs.com/package/@yahyahammad/venastine"><img alt="npm package @yahyahammad/venastine" src="https://img.shields.io/npm/v/%40yahyahammad%2Fvenastine?logo=npm&label=npm"></a>
|
|
13
|
+
<a href="https://github.com/YahyaHammad/Venastine-Research-Harness/actions/workflows/tests.yml"><img alt="Tests on main" src="https://img.shields.io/github/actions/workflow/status/YahyaHammad/Venastine-Research-Harness/tests.yml?branch=main&label=tests&logo=githubactions&logoColor=white"></a>
|
|
14
|
+
<a href="#setup"><img alt="Python 3.11, 3.12 and 3.13" src="https://img.shields.io/badge/python-3.11%20%7C%203.12%20%7C%203.13-3776AB?logo=python&logoColor=white"></a>
|
|
15
|
+
<a href="./LICENSE"><img alt="License: Apache-2.0" src="https://img.shields.io/badge/license-Apache--2.0-F5DE53"></a>
|
|
16
|
+
</p>
|
|
17
|
+
|
|
18
|
+

|
|
19
|
+
|
|
5
20
|
An agentic harness built from scratch — no LangChain, no agent framework — with two modes that share the same machinery:
|
|
6
21
|
|
|
7
22
|
- **Chat** — a normal tool-using assistant loop.
|
|
@@ -122,6 +137,40 @@ Two things differ from a clone, both to make a command you run from anywhere beh
|
|
|
122
137
|
|
|
123
138
|
`venastine --venastine-doctor` prints every one of those resolved paths, plus the interpreter and environment it found.
|
|
124
139
|
|
|
140
|
+
#### With yarn, bun or pnpm
|
|
141
|
+
|
|
142
|
+
The same package installs from the npm registry with the other package
|
|
143
|
+
managers, and everything above about the launcher applies to each. The
|
|
144
|
+
launcher still needs Node.js 18 or newer, and the harness Python 3.11 or newer.
|
|
145
|
+
|
|
146
|
+
```bash
|
|
147
|
+
yarn global add @yahyahammad/venastine
|
|
148
|
+
```
|
|
149
|
+
|
|
150
|
+
```bash
|
|
151
|
+
bun add -g --backend=copyfile @yahyahammad/venastine
|
|
152
|
+
```
|
|
153
|
+
|
|
154
|
+
```bash
|
|
155
|
+
pnpm add -g --config.package-import-method=copy @yahyahammad/venastine
|
|
156
|
+
```
|
|
157
|
+
|
|
158
|
+
To update, run the same command with `@latest` after the package name.
|
|
159
|
+
|
|
160
|
+
* **yarn means Yarn 1 (classic).** Yarn 2 and later have no global install.
|
|
161
|
+
* **bun and pnpm need the copy flag, on updates too.** By default both
|
|
162
|
+
hardlink the package's files out of a shared cache or store, so the
|
|
163
|
+
harness's own source, which it protects from tool writes, would have a
|
|
164
|
+
second name in a folder it does not protect. The flag makes them copy,
|
|
165
|
+
as npm and yarn do.
|
|
166
|
+
* **pnpm does not carry `config.yaml` edits across an update.** It puts
|
|
167
|
+
every install in a new directory, and the harness keys your saved
|
|
168
|
+
`config.yaml` values by the install's directory, so after an update the
|
|
169
|
+
values you set with `/config` or by editing that file are back at the
|
|
170
|
+
shipped defaults. What lives in `~/.config/venastine` (`settings.json`,
|
|
171
|
+
keys, conversations and memories) is unaffected. If you rely on
|
|
172
|
+
`config.yaml` edits, install with npm, yarn or bun instead.
|
|
173
|
+
|
|
125
174
|
### From a clone
|
|
126
175
|
|
|
127
176
|
```bash
|
|
@@ -186,7 +235,7 @@ Every launch flag is tabulated in the [CLI flag reference](#cli-flag-reference)
|
|
|
186
235
|
|
|
187
236
|
A turn prints as it runs, one step at a time: the model's reasoning, what it said, and a line for each tool call as it starts. During a turn, Ctrl+C twice within two seconds stops it at the next safe point; the first press only says so, so a stray one costs nothing. At the prompt, Ctrl+C or Ctrl+D exits. The thread id is printed after the first response so you can resume later — and resuming **replays the conversation**, so you can see what you are picking up. Tool calls replay as one line each; their results do not, since a single research turn can carry hundreds of kilobytes of fetched pages.
|
|
188
237
|
|
|
189
|
-
A line that starts with `/` is a command, as in the TUI, and never reaches the model; an unknown one is an error rather than a chat turn. The CLI has
|
|
238
|
+
A line that starts with `/` is a command, as in the TUI, and never reaches the model; an unknown one is an error rather than a chat turn. The CLI has twenty: `/help`, `/quit`, `/threads`, `/resume`, `/new`, `/goal`, `/agent`, `/skill`, `/grill-me`, `/effort`, `/model`, `/compact`, `/summary`, `/ref`, `/memories`, `/forget`, `/claims`, `/call`, `/network` and `/classifier`. They mean what the TUI's do: both shells read one catalog of commands, so a name, its arguments and its description cannot differ between them. `/threads` and `/ref` print a numbered list and ask for a number, or a thread id for one the list does not show. `/call` lists this conversation's tool calls, numbered, and `/call 3` shows the third in full: the whole command and what it returned, or a background session's output, or the run a subagent made. It is the CLI's version of ctrl+clicking a tool line in the TUI. `/model` here lasts for the session, since the CLI starts from `--model` or settings.json.
|
|
190
239
|
|
|
191
240
|
---
|
|
192
241
|
|
|
@@ -440,6 +489,8 @@ Because approval ORs, an override can only ever *add* a prompt. There is no way
|
|
|
440
489
|
| Depends on the call | `read`, `view_image`, `write`, `edit` — decided per path, and all four are denied by default anyway. `view_image` is also only offered where the image can reach the model: a provider marked `accepts_images` in `providers.json`, outside a research pass |
|
|
441
490
|
| Always | `shell` and the three that start a session (`shell_background`, `shell_monitor`, `shell_interactive`), `spawn_subagent`, `remember`, `write_project_doc`, and **every MCP tool** |
|
|
442
491
|
|
|
492
|
+
**"Decided per path" is three rules.** Inside the workspace (`./workspace` under the directory you launched from, unless `AGENT_WORKSPACE` or `workspace_dir` in `config.yaml` moves it), a call runs unasked. Outside it, a call asks every time, and that includes the rest of your project while the workspace is its `workspace` folder. A path through a `.venastine/` directory is refused outright, wherever it lands and whichever name reaches it, symlink or hard link. So enabling the file tools and launching with `AGENT_WORKSPACE` set to your project is what lets the model change any file there without asking.
|
|
493
|
+
|
|
443
494
|
Every registered tool appears in that table, and a test asserts it (audit #125): `ToolPermissions` and `ToolApprovals` are plain dataclasses, so "did the table keep up with the registry" is a question the suite can answer instead of a reader. `write_project_doc` is the one the omission mattered for — it is what `/init` uses to write documents into a repository, and §24's I1 gives it a fixed allowlist of document *names* with no path parameter precisely because it is dangerous enough to constrain.
|
|
444
495
|
|
|
445
496
|
`ask_user` is ungated for a reason worth stating, since a table of default approvals is exactly where it belongs: an approval-gated tool is not *advertised* where nothing can ask, so gating the tool whose entire job is asking would make it invisible rather than deniable.
|
|
@@ -454,9 +505,12 @@ Every registered tool appears in that table, and a test asserts it (audit #125):
|
|
|
454
505
|
|---|---|
|
|
455
506
|
| `always` | every command is asked about, whatever it does |
|
|
456
507
|
| `tiered` | **shipped.** The classifier decides — see below. Only a read-only command runs unasked; anything that can run code asks, even inside the container |
|
|
508
|
+
| `classified` | `tiered`, plus a **decision model** you configure, asked about a single command that would run in the container and that `tiered` would ask about. It runs unasked only if the model's verdict passes all four of your thresholds. Compound commands, reads outside the workspace, the host fallback, WSL and SSH are never sent to it and ask as under `tiered`. Flagged on the startup banner and the TUI badge |
|
|
457
509
|
| `contained` | anything the container confines runs unasked unless it needs network, code included; reads outside the workspace and network commands still ask. Flagged on the startup banner and the TUI badge |
|
|
458
510
|
| `never` | nothing is ever asked about |
|
|
459
511
|
|
|
512
|
+
**`classified` widens what runs without asking you, and that is the point of it.** It is opt-in and ships unconfigured. The `shell_classifier_*` keys in `config.yaml` name the model: TypeSafe's Jev (`https://api.typesafe.ai/v1`), a decision model under Ollama 0.35 or later (`http://localhost:11434/v1`, for example `nimble`), or any OpenAI-compatible model asked for a strict JSON answer. The key comes from a `providers.json` entry you name, never from `config.yaml`. The model is shown the command and the project path, and never the agent's stated reason. It answers four questions: how contained the command's effects are, how risky it is, how much it changes your project, and whether it uses the network. Each answer has a threshold from 0 to 1. A command that fails one asks you, with the verdict in the prompt (`shell_classifier_on_fail: ask`, shipped), or is refused without asking (`reject`). No answer in time, an error, or a malformed answer always asks. The model's guarantee is that its answer has the right type, not that its judgement is right, and the command it judges is text the agent wrote. So start with `ask` and read its verdicts before trusting it. `/classifier` shows the settings and the verdicts held this session; `/classifier test <command>` asks the model without running anything, under any mode; `/classifier clear` asks it afresh.
|
|
513
|
+
|
|
460
514
|
There is deliberately no second switch: `tool_approvals` has no `shell` key, so nothing here can disagree with the mode. An agent's `approval_overrides` still reaches shell with the same one-way power -- an agent can demand approval for a command the mode would auto-approve, and can never wave one past a gate the mode set. An unknown mode string raises at startup rather than falling back to a default — one direction of that default asks about everything and the other about nothing, and a typo cannot pick.
|
|
461
515
|
|
|
462
516
|
Under `tiered` and `contained`, each command is classified **once** into what it can do, and the same answer is read by the approval check and by the sandbox that runs it:
|
|
@@ -650,9 +704,12 @@ Agent frontmatter fields:
|
|
|
650
704
|
| `use_memory` | bool | `true` | Inject durable memories (plain chat counts as opted in) |
|
|
651
705
|
| `max_steps` | int ≥ 1 | caller's ceiling | **Replaces** the caller's ceiling rather than narrowing it — a value above `MAX_ITERATIONS` raises it, making this the one frontmatter field that can widen anything; an invalid value is repaired to the default with a warning naming what it read |
|
|
652
706
|
| `spawnable` | bool | `false` for your files | Whether `spawn_subagent` may feed this agent |
|
|
707
|
+
| `spawn_targets` | list | any spawnable agent | Which agents this one may spawn. It only ever narrows, and narrows again for anything it spawns in turn, as described above |
|
|
653
708
|
|
|
654
709
|
Skill frontmatter fields: `name` (required), `description`, and `additional_tools` — a list of tool names stating what the skill's methodology *depends on*. It grants nothing: activation proceeds and tells you what is missing under the current agent.
|
|
655
710
|
|
|
711
|
+
A key that neither list names is **ignored, not refused**, in either kind of file. A field spelled the way another harness spells it, such as `tools:` for `allowed_tools`, therefore does nothing at all; a test holds both lists to the keys the loader reads, so neither can fall behind it.
|
|
712
|
+
|
|
656
713
|
### A description is text, and the prompt treats it as one line
|
|
657
714
|
|
|
658
715
|
An agent's or a skill's `name` and `description` go straight into the system prompt, so they are collapsed to a single line and capped at 300 characters when the file is read. Without that, a description written as a multi-line YAML block escapes its bullet and renders as a section of the prompt in its own right — indistinguishable from one this harness wrote. The cap is generous next to real ones: the longest description shipped here is 171 characters.
|
|
@@ -710,6 +767,7 @@ paths are cached and 200 matches offered; narrow the query when a limit is shown
|
|
|
710
767
|
| `/trigger [tokens\|off]` | Override **when this chat compacts**, for this session only. An absolute prompt size (`80k`), not a margin below the window. A value below the warning margin or the keep-recent floor is refused naming it; a value below the thread's current size is accepted and says a fold is coming |
|
|
711
768
|
| `/window [tokens\|off]` | Set this model's **context window** and remember it, per `(provider, model)` and across launches. It outranks both the provider's own answer and the built-in table, and it moves everything derived from the window: the chat compaction trigger, the research-pass backstop and the summarizer's input budget. Above what the provider reports it warns rather than refusing, because raising it is the point — a gateway or a self-hosted endpoint can report its own default rather than your deployment's |
|
|
712
769
|
| `/network` | What each network meter has used against its ceiling over the rolling window, which part is an estimate, and when room returns. Every ceiling is off until you set one in `config.yaml`; see [Budgets and stop conditions](#budgets-and-stop-conditions) |
|
|
770
|
+
| `/classifier [test <command>\|clear]` | The shell's decision model under `shell_approval_mode: classified`: its settings, which `providers.json` entry its key comes from, its thresholds and the verdicts held this session. `test` asks it about a command without running anything, under any mode; `clear` drops the held verdicts so it is asked again |
|
|
713
771
|
| `/config [key] [value]` | Browse and set the harness's own settings without leaving the session. Typing `/config ` lists all 133 of them in the same panel the slash commands use, with the values each one takes and the value in the file; completing a key then offers that key's values. A value you have already changed reads `pending restart` until you relaunch, and bare `/config` lists every one of them, old to new — so several settings can be changed in a row and applied together. Bare `/config <key>` explains one, including what outranks it — an environment variable, a `settings.json` key — and containers are listed but edited in the file. A write goes through the whole document's validation first, keeps every comment in `config.yaml`, and offers to relaunch, because values are read once at startup. The nine AUTHORITY keys ask first, naming what the key permits |
|
|
714
772
|
| `/claims [run id]` | Open the claims view — tier, grounding status, assumption flags, score breakdown per claim — during a run or afterwards (**ctrl+l**) |
|
|
715
773
|
| `/copy [last\|report\|claims\|conversation\|all] [--file <path>]` | Copy out of the session. Defaults to the last response; `conversation` is the exchange alone — your turns, the answers, the reasoning, the tool calls and the diffs; `all` is everything on screen (the harness's own lines included) plus report plus claims. Clipboard delivery cannot be confirmed, so `--file <path>` is the route that provably worked |
|
|
@@ -936,7 +994,8 @@ Deliberately not settings.json keys: editing these means editing the harness's o
|
|
|
936
994
|
| `google_thinking_budgets` | low→2,048 … max→dynamic | Google level→thinking-token map |
|
|
937
995
|
| `models_rejecting_sampling_params` | current Anthropic models, and the two Gemini models that ignore them | The Anthropic models 400 on temperature/top_p/top_k and the Gemini ones ignore them; such parameters are dropped with a WARNING rather than sent. OpenAI's reasoning models are handled by effort, not by this list |
|
|
938
996
|
| `model_context_windows` / `default_context_window` | 256k fallback | The **fallback** for the window, not the only source: Anthropic and Google report it on their model endpoints, and the OpenAI-compatible providers that carry it (Groq, Mistral, Together, OpenRouter) are read through one alias sniff. This table answers for the ones that report nothing (OpenAI, DeepSeek, Perplexity). Feeds the research-pass compaction backstop **and** the summarizer's one-call input budget; an unknown model warns once and assumes the default. Keys are stored normalized — no date suffix, no `vendor/` prefix |
|
|
939
|
-
| `shell_approval_mode` | `tiered` | The shell gate: `always` / `tiered` / `never`; a bad value raises at import. Rejected in settings.json by name, see above |
|
|
997
|
+
| `shell_approval_mode` | `tiered` | The shell gate: `always` / `tiered` / `classified` / `contained` / `never`; a bad value raises at import. Rejected in settings.json by name, see above |
|
|
998
|
+
| `shell_classifier_*` | unconfigured; thresholds 0.75 / 0.25 / 0.5 / 0.5 | The decision model `classified` asks: style, URL, model, the `providers.json` entry its key comes from, how long to wait, what a failure does, and its four thresholds. All ten are authority keys |
|
|
940
999
|
| `network_allowed_commands` | pip, curl, git, npm, … | Binaries granted network access inside the sandbox. Matched against **every** word of a command that needs a sandbox, so `cd x && pip install .` is recognised and asked about — and against the **first word only** of an inert one, which cannot chain, so `grep pip notes.txt` still runs unprompted. This list is a *lexical* test, so since ROADMAP_v3 §50 the agent can also declare `requires_network: true` on `shell`, `shell_background`, `shell_monitor` and `shell_interactive` for anything the word list cannot see — an inline script that opens a socket, say. The declaration only ever ADDS network access, and the command is then asked about wherever your approval mode would allow a prompt |
|
|
941
1000
|
| `inert_commands` | ls, cat, grep, wc, … | Read-only commands eligible to run unprompted under `tiered` — as a plain argument list in the container, or on the host only when no container runtime is available |
|
|
942
1001
|
| Sandbox bounds | image `docker.io/library/python:3.13-slim`; 120 s, 2048 MB, 1 CPU core, 200 pids — under Docker, or Podman when Docker cannot run the sandbox. The container's CPU control is a RATE (`--cpus 1`), so `sandbox_cpu_seconds` (30 CPU-s) is a budget the weak fallback enforces instead, and on Unix only | `sandbox_docker_image`, `sandbox_timeout_seconds`, `sandbox_memory_mb`, `sandbox_cpu_seconds`, `sandbox_max_pids` |
|
|
@@ -999,15 +1058,18 @@ Deliberately not settings.json keys: editing these means editing the harness's o
|
|
|
999
1058
|
|
|
1000
1059
|
**Built:** ROADMAP.md §1–§12 (§10's ensemble mode rebuilt by its revisit — diversity now comes from a roster of different models) and ROADMAP_v2.md §13 (streaming loop), §14 (config loader + workspace trust), §15 (permissions), §16 (TUI), §17 (MCP), §18 (agents), §19 (skills), §20 (post-pipeline review), §21a (compaction + `pin`), §21b (durable memory), §22 (live pipeline progress), §25 (authorised tool use in the pipeline), §26 (research legibility: per-pass tool calls, code-stage announcements, the claims view, role colour, `/copy`), §27 (thread legibility: transcript replay on resume, a conversations-only thread picker), §21c (session summaries and cross-thread referencing — §21 is now complete), §24 (`/init`), §23 (interactive tools: the response channel, per-tool subagent sign-off, `ask_user`, and the todo list), §28 (the shell capability classifier).
|
|
1001
1060
|
|
|
1002
|
-
**Remaining:** nothing in `ROADMAP.md` or `ROADMAP_v2.md`, and §10's revisit is now closed. `ROADMAP_v3.md` §49–§
|
|
1061
|
+
**Remaining:** nothing in `ROADMAP.md` or `ROADMAP_v2.md`, and §10's revisit is now closed. `ROADMAP_v3.md` §49–§59 and §61–§93 are built. The section still to build is §60 (Unified Harness Protocol compliance), specified and awaiting detailed design. §56 lets a decision model approve a single contained shell command under the opt-in `classified` mode; §54 completes project file/folder mentions without loading contents; §58 supplies a short harness identity that follows the current model; §83's final live checks are confirmed, and the ChatGPT header distinguishes a selected provider from an active sign-in. `TECHNICAL_DEBT.md` items 14, 17, 18, 21, 102 and 116 are open, and so is the secrets half of item 97. The two checks recorded against §10 are answered from the providers' documentation in `ROADMAP_v3.md` §87: the default `temperature` on OpenAI and Google is 1.0, and OpenAI's reasoning models refuse one whenever reasoning effort is not `none`, which the guard now follows.
|
|
1003
1062
|
|
|
1004
|
-
**
|
|
1005
|
-
|
|
1006
|
-
mistake this project keeps recording against itself
|
|
1007
|
-
**
|
|
1008
|
-
`
|
|
1009
|
-
|
|
1010
|
-
|
|
1063
|
+
**And both audits are closed.** Both roadmaps being built is not the same as the code
|
|
1064
|
+
being clean, and saying only the first would be the "built and runs are different claims"
|
|
1065
|
+
mistake this project keeps recording against itself, so the audits get their own line.
|
|
1066
|
+
**Audit Pass 1** (tracker `#15`) is **154 issues, all closed**: 130 findings (8 `S1`, plus 4
|
|
1067
|
+
filed as `S0` before the issue forms; 30 `S2`, 79 `S3` and 9 `S4`), 22 unit reviews, one
|
|
1068
|
+
enhancement and the tracker. **The review marathon** (`docs/REVIEW_MARATHON.md`) filed the
|
|
1069
|
+
findings it deferred as 59 more issues, `#203`–`#261` under tracker `#262`: 17 `S3` and
|
|
1070
|
+
42 `S4`, **all closed**. The last finding of either closed on 2026-11-04. Both were worked in
|
|
1071
|
+
numbered fix batches, each recorded in `DEVLOG.md` and `tests/BREAKING_CHANGES.md` with what
|
|
1072
|
+
was measured and what would break the fix.
|
|
1011
1073
|
|
|
1012
1074
|
**#157 — `shell`'s unbounded auto-approval — is closed** by batch 8 and ROADMAP_v2 §28; the
|
|
1013
1075
|
classifier is described under *Security model* above. If you have a fork or a local config,
|
|
@@ -1015,11 +1077,13 @@ note that shell approval is governed solely by `shell_approval_mode` -- `tool_ap
|
|
|
1015
1077
|
deliberately has no `shell` key, so an old file carrying one is refused at startup
|
|
1016
1078
|
naming the key — see `tests/BREAKING_CHANGES.md` §24.
|
|
1017
1079
|
|
|
1018
|
-
Run the test suite with `pytest` —
|
|
1080
|
+
Run the test suite with `pytest` — 9310 tests, fully offline, no API keys needed. Since batch 243 the suite holds itself to that: a test that asks DNS for a name, or writes into your home folder, fails by name, and every test process runs with HOME pointing at a throwaway folder of its own. The suite's own packages are `requirements-test.txt`, separate from `requirements.txt` since batch 136: they used to be in the shipped dependency list, so installing the harness also installed a test runner. It runs in parallel by default (`pytest-xdist`, 70% of your cores): **1:26 on a 16-core machine, against 9:06 single-process.** `pytest -n0` forces the old serial run, which is what you want under a debugger or when something is parsing the output. Eight further tests are marked `integration` and excluded by default: three spawn a real stdio MCP server, one runs an external program through the host fallback shell, one builds the hosted bundle's Docker image, and three run the installers' uv path against a real uv (`pytest -m integration`).
|
|
1019
1081
|
|
|
1020
1082
|
## Documentation
|
|
1021
1083
|
|
|
1022
|
-
This file is the user-facing one.
|
|
1084
|
+
This file is the user-facing one. The [wiki](https://github.com/YahyaHammad/Venastine-Research-Harness/wiki) arranges it by task: installation, getting started, what each tool may do and when it asks, the shell and its sandbox, writing agents and skills, MCP servers, and an FAQ. It lives outside the repository, so no test checks it: each page names the release it was checked against, and where a page and this file disagree, this file is the reference.
|
|
1085
|
+
|
|
1086
|
+
For working on the code:
|
|
1023
1087
|
|
|
1024
1088
|
- **[AGENTS.md](./AGENTS.md)** — the agent-context file, and the place to start. `CLAUDE.md` and `QWEN.md` are pointers to it so a tool looking for a particular filename finds one without the guidance being duplicated.
|
|
1025
1089
|
- **[ARCHITECTURE.md](./docs/ARCHITECTURE.md)** — file-by-file contracts, what belongs where, and the known gotchas. Read before changing anything non-trivial, especially around persistence (`database.py` / `storage.py` / `core/memory.py` are three distinct and easily confused responsibilities).
|
|
@@ -1034,6 +1098,12 @@ This file is the user-facing one. For working on the code:
|
|
|
1034
1098
|
- **[CODE_OF_CONDUCT.md](./docs/CODE_OF_CONDUCT.md)** — Contributor Covenant 2.1, and the private channel for reporting a violation.
|
|
1035
1099
|
- **[THIRD_PARTY_NOTICES.md](./docs/THIRD_PARTY_NOTICES.md)** — the dependency license assessment: every declared direct dependency, its license, and its Apache-2.0 compatibility.
|
|
1036
1100
|
|
|
1101
|
+
## Citing
|
|
1102
|
+
|
|
1103
|
+
If you use Venastine Research Harness in your research, please cite it with the metadata in
|
|
1104
|
+
[CITATION.cff](./CITATION.cff). GitHub turns that file into the **Cite this repository** link in
|
|
1105
|
+
the repository's sidebar, which gives the citation in APA and BibTeX to copy.
|
|
1106
|
+
|
|
1037
1107
|
## License
|
|
1038
1108
|
|
|
1039
1109
|
Apache-2.0 — see [LICENSE](./LICENSE) and [NOTICE](./NOTICE). Copyright 2026 Yahya Hammad.
|
package/cli_commands.py
CHANGED
|
@@ -663,6 +663,23 @@ def _cmd_memories(session: ChatSession, args: str) -> None:
|
|
|
663
663
|
print_memories()
|
|
664
664
|
|
|
665
665
|
|
|
666
|
+
def _cmd_classifier(session: ChatSession, args: str) -> None:
|
|
667
|
+
"""ROADMAP_v3 §56 (JV19). The TUI's /classifier, line for line; `test`
|
|
668
|
+
runs on the chat loop's thread here, which is where a CLI command
|
|
669
|
+
runs."""
|
|
670
|
+
from core import decision_classifier
|
|
671
|
+
|
|
672
|
+
word, _, rest = args.strip().partition(" ")
|
|
673
|
+
if word == "test":
|
|
674
|
+
command = rest.strip()
|
|
675
|
+
lines = (decision_classifier.test_lines(command) if command
|
|
676
|
+
else [decision_classifier.USAGE])
|
|
677
|
+
else:
|
|
678
|
+
lines = decision_classifier.command_lines(args)
|
|
679
|
+
for line in lines:
|
|
680
|
+
_out(line)
|
|
681
|
+
|
|
682
|
+
|
|
666
683
|
def _cmd_network(session: ChatSession, args: str) -> None:
|
|
667
684
|
"""ROADMAP_v3 §53 (IO13). The TUI's /network, line for line: both
|
|
668
685
|
print `_net_common.network_report`."""
|
|
@@ -824,6 +841,7 @@ def register_cli_commands() -> None:
|
|
|
824
841
|
bind("agent", _cmd_agent),
|
|
825
842
|
bind("call", _cmd_call),
|
|
826
843
|
bind("claims", _cmd_claims),
|
|
844
|
+
bind("classifier", _cmd_classifier),
|
|
827
845
|
bind("compact", _cmd_compact),
|
|
828
846
|
bind("effort", _cmd_effort),
|
|
829
847
|
bind("forget", _cmd_forget),
|
package/config.yaml
CHANGED
|
@@ -739,6 +739,13 @@ ssh_binary: ''
|
|
|
739
739
|
# tiered -- read-only commands from inert_commands, every argument
|
|
740
740
|
# inside the workspace, run unprompted; every other command
|
|
741
741
|
# asks, including anything that can run code
|
|
742
|
+
# classified -- tiered, plus a decision model (the shell_classifier_*
|
|
743
|
+
# keys below) asked about a single command that runs in the
|
|
744
|
+
# container and that tiered would ask about. If it passes
|
|
745
|
+
# your thresholds, the command runs unprompted. Compound
|
|
746
|
+
# commands (any of ; | & $ ` > < ( ) { } ! # ~ ' " \),
|
|
747
|
+
# reads outside the workspace, the host fallback, WSL and
|
|
748
|
+
# SSH are never sent to it and ask exactly as under tiered
|
|
742
749
|
# contained -- anything that runs in the container without network runs
|
|
743
750
|
# unprompted, code included; reads outside the workspace and
|
|
744
751
|
# network commands still ask
|
|
@@ -751,6 +758,73 @@ ssh_binary: ''
|
|
|
751
758
|
# no `shell` key, so there is no second switch that can disagree with this.
|
|
752
759
|
shell_approval_mode: tiered
|
|
753
760
|
|
|
761
|
+
# AUTHORITY -- the decision model `classified` asks (ROADMAP_v3 §56). Only
|
|
762
|
+
# read when shell_approval_mode is `classified`; `/classifier test <command>`
|
|
763
|
+
# shows a verdict under any mode without running anything.
|
|
764
|
+
#
|
|
765
|
+
# READ THIS BEFORE TURNING IT ON. This is the one setting in the harness
|
|
766
|
+
# that WIDENS what runs without asking you, by design. The model reads the
|
|
767
|
+
# command and the project path and nothing else -- never the agent's stated
|
|
768
|
+
# reason -- but the command itself is text the agent wrote, and the agent
|
|
769
|
+
# may have read something written to steer it. A decision model's
|
|
770
|
+
# guarantee is that its answer has the right TYPE (a number in range), not
|
|
771
|
+
# that its judgement is right; TypeSafe's own documentation says content
|
|
772
|
+
# written to steer Jev "can move the answer". A confident wrong number is
|
|
773
|
+
# exactly what a threshold acts on. Start with shell_classifier_on_fail:
|
|
774
|
+
# ask, and watch its verdicts in the approval prompt before trusting it.
|
|
775
|
+
#
|
|
776
|
+
# A small local model and a hosted one are two different policies wearing
|
|
777
|
+
# the same thresholds. And a model ALIAS moves when its vendor ships a new
|
|
778
|
+
# release (TypeSafe: jev-latest), so name a version (jev-1.14.0) once you
|
|
779
|
+
# have tuned the numbers against it.
|
|
780
|
+
#
|
|
781
|
+
# Which wire to speak: `systemone` (TypeSafe's Jev at
|
|
782
|
+
# https://api.typesafe.ai/v1, or Ollama 0.35+ at http://localhost:11434/v1
|
|
783
|
+
# with a decision model such as nimble), or `openai_json` (any
|
|
784
|
+
# OpenAI-compatible provider, asked for a strict JSON answer; its numbers
|
|
785
|
+
# are coarser, and not calibrated probabilities).
|
|
786
|
+
shell_classifier_style: systemone
|
|
787
|
+
|
|
788
|
+
# AUTHORITY -- the classifier's base URL, ending at /v1. Empty means not
|
|
789
|
+
# configured: every command the model would have answered asks, exactly as
|
|
790
|
+
# under tiered.
|
|
791
|
+
shell_classifier_url: ''
|
|
792
|
+
|
|
793
|
+
# AUTHORITY -- the model to ask, e.g. jev-1.14.0 or nimble.
|
|
794
|
+
shell_classifier_model: ''
|
|
795
|
+
|
|
796
|
+
# AUTHORITY -- the providers.json ENTRY whose API_KEY is sent to the URL
|
|
797
|
+
# above, e.g. TYPESAFE or OPENAI. Empty sends no key (a local Ollama needs
|
|
798
|
+
# none). The key itself never goes in this file, and it is never sent
|
|
799
|
+
# over plain http to another machine.
|
|
800
|
+
shell_classifier_credential: ''
|
|
801
|
+
|
|
802
|
+
# AUTHORITY -- seconds to wait for a verdict. No verdict in time means the
|
|
803
|
+
# command asks; it never means it runs.
|
|
804
|
+
shell_classifier_timeout_s: 10
|
|
805
|
+
|
|
806
|
+
# AUTHORITY -- what a command outside any threshold does: `ask` raises the
|
|
807
|
+
# normal approval prompt, with the verdict shown in it; `reject` refuses
|
|
808
|
+
# the command without asking. A model that gives no verdict always asks.
|
|
809
|
+
shell_classifier_on_fail: ask
|
|
810
|
+
|
|
811
|
+
# AUTHORITY -- the four thresholds, each from 0 to 1. A command runs
|
|
812
|
+
# unprompted only if all four pass. This one, containedness: how
|
|
813
|
+
# completely its effects stay inside the project. Passes at or ABOVE it.
|
|
814
|
+
shell_classifier_min_containedness: 0.75
|
|
815
|
+
|
|
816
|
+
# AUTHORITY -- general risk: how much harm the command could do. Passes at
|
|
817
|
+
# or below it.
|
|
818
|
+
shell_classifier_max_risk: 0.25
|
|
819
|
+
|
|
820
|
+
# AUTHORITY -- workspace modification: how much it changes the project's
|
|
821
|
+
# files. Passes at or below it.
|
|
822
|
+
shell_classifier_max_modification: 0.5
|
|
823
|
+
|
|
824
|
+
# AUTHORITY -- networkedness: the probability it uses the network. Passes
|
|
825
|
+
# at or below it.
|
|
826
|
+
shell_classifier_max_network: 0.5
|
|
827
|
+
|
|
754
828
|
# The image sandboxed commands run in, under Docker or Podman (whichever
|
|
755
829
|
# runs the sandbox -- ROADMAP_v3 §49, SS22). Fully qualified so Podman
|
|
756
830
|
# resolves it without a short-name alias (SS23). AGENT_SANDBOX_IMAGE
|
package/config_edit.py
CHANGED
|
@@ -88,11 +88,48 @@ TABLES = ("tool_permissions", "tool_approvals")
|
|
|
88
88
|
#: `tests/test_config_edit.py` holds this against HARNESS_AUTHORITY_KEYS.
|
|
89
89
|
AUTHORITY_EFFECT = {
|
|
90
90
|
"shell_approval_mode": (
|
|
91
|
-
"how often a shell command stops to ask you. `
|
|
91
|
+
"how often a shell command stops to ask you. `classified` lets a "
|
|
92
|
+
"decision model run a single contained command without asking "
|
|
93
|
+
"when it passes your thresholds, `contained` runs any "
|
|
92
94
|
"code the agent writes inside the container without asking, and "
|
|
93
95
|
"`never` runs every command the agent writes without asking, "
|
|
94
96
|
"including in a session where untrusted content is already in the "
|
|
95
97
|
"context window."),
|
|
98
|
+
# ROADMAP_v3 §56 (JV8). Each sentence says what moving the key lets
|
|
99
|
+
# through, because the confirmation exists to say that.
|
|
100
|
+
"shell_classifier_style": (
|
|
101
|
+
"which wire the shell classifier speaks. Under `classified`, its "
|
|
102
|
+
"answer decides whether a command runs without asking you."),
|
|
103
|
+
"shell_classifier_url": (
|
|
104
|
+
"which server decides, under `classified`, whether a shell command "
|
|
105
|
+
"runs without asking you. Every command it is asked about is sent "
|
|
106
|
+
"there."),
|
|
107
|
+
"shell_classifier_model": (
|
|
108
|
+
"which model decides, under `classified`, whether a shell command "
|
|
109
|
+
"runs without asking you."),
|
|
110
|
+
"shell_classifier_credential": (
|
|
111
|
+
"which providers.json key is sent to the shell classifier's URL "
|
|
112
|
+
"with every command it is asked about."),
|
|
113
|
+
"shell_classifier_timeout_s": (
|
|
114
|
+
"how long a shell command waits for the classifier. No answer in "
|
|
115
|
+
"time asks you; it never runs the command."),
|
|
116
|
+
"shell_classifier_on_fail": (
|
|
117
|
+
"what a shell command the classifier fails does: `ask` asks you, "
|
|
118
|
+
"`reject` refuses it without asking."),
|
|
119
|
+
"shell_classifier_min_containedness": (
|
|
120
|
+
"how contained a command must look to the classifier to run "
|
|
121
|
+
"without asking you. Lower lets more through."),
|
|
122
|
+
"shell_classifier_max_risk": (
|
|
123
|
+
"how risky a command may look to the classifier and still run "
|
|
124
|
+
"without asking you. Higher lets more through."),
|
|
125
|
+
"shell_classifier_max_modification": (
|
|
126
|
+
"how much a command may change your project, by the classifier's "
|
|
127
|
+
"judgement, and still run without asking you. Higher lets more "
|
|
128
|
+
"through."),
|
|
129
|
+
"shell_classifier_max_network": (
|
|
130
|
+
"how likely a command may be to use the network, by the "
|
|
131
|
+
"classifier's judgement, and still run without asking you. Higher "
|
|
132
|
+
"lets more through."),
|
|
96
133
|
"allow_insecure_sandbox_fallback": (
|
|
97
134
|
"whether a shell command may run WITHOUT the container when neither "
|
|
98
135
|
"Docker nor Podman can run one -- on the host, with your files."),
|
package/config_schema.py
CHANGED
|
@@ -156,7 +156,30 @@ NonNegativeNumber = Annotated[float, BeforeValidator(_number_only),
|
|
|
156
156
|
#: startup failure and never a policy that quietly means something else --
|
|
157
157
|
#: one direction of a fallback asks about everything and the other about
|
|
158
158
|
#: nothing, and a typo cannot pick.
|
|
159
|
-
ShellApprovalMode = Literal["always", "tiered", "
|
|
159
|
+
ShellApprovalMode = Literal["always", "tiered", "classified", "contained",
|
|
160
|
+
"never"]
|
|
161
|
+
|
|
162
|
+
#: ROADMAP_v3 §56 (JV7): which wire the decision model speaks, named and
|
|
163
|
+
#: never probed -- a probe is a network call at launch and a guess about a
|
|
164
|
+
#: security control.
|
|
165
|
+
ShellClassifierStyle = Literal["systemone", "openai_json"]
|
|
166
|
+
|
|
167
|
+
#: ROADMAP_v3 §56 (JV6, JV18): what a call outside a threshold does.
|
|
168
|
+
ShellClassifierOnFail = Literal["ask", "reject"]
|
|
169
|
+
|
|
170
|
+
|
|
171
|
+
def _http_url_or_empty(value: Any) -> Any:
|
|
172
|
+
"""Empty, or an http(s) URL. A typo in the endpoint that decides
|
|
173
|
+
whether a human is asked is a startup failure, not a classifier that
|
|
174
|
+
silently errors on every call."""
|
|
175
|
+
if isinstance(value, str) and value.strip() and not (
|
|
176
|
+
value.startswith("https://") or value.startswith("http://")):
|
|
177
|
+
raise ValueError("must be empty or start with https:// or http://")
|
|
178
|
+
return value
|
|
179
|
+
|
|
180
|
+
|
|
181
|
+
#: `shell_classifier_url`: empty (not configured) or an http(s) base URL.
|
|
182
|
+
ClassifierUrl = Annotated[str, BeforeValidator(_http_url_or_empty)]
|
|
160
183
|
|
|
161
184
|
#: "rederive" / "chain". Validated here AND in
|
|
162
185
|
#: `config_loader.effective_compaction`, which checks the value a
|
|
@@ -440,6 +463,22 @@ class HarnessConfig(_Model):
|
|
|
440
463
|
ssh_host_key: str
|
|
441
464
|
ssh_binary: str
|
|
442
465
|
shell_approval_mode: ShellApprovalMode
|
|
466
|
+
# ROADMAP_v3 §56 (JV8, JV11). The decision model the `classified` mode
|
|
467
|
+
# asks. Every one is an authority key: the URL decides who answers,
|
|
468
|
+
# the credential NAME decides which providers.json key is sent there,
|
|
469
|
+
# and the four numbers decide whether a human is asked. Flat scalars,
|
|
470
|
+
# the SSH backend's shape (SS36): one classifier, a fact about this
|
|
471
|
+
# machine and not about any one command.
|
|
472
|
+
shell_classifier_style: ShellClassifierStyle
|
|
473
|
+
shell_classifier_url: ClassifierUrl
|
|
474
|
+
shell_classifier_model: str
|
|
475
|
+
shell_classifier_credential: str
|
|
476
|
+
shell_classifier_timeout_s: PositiveNumber
|
|
477
|
+
shell_classifier_on_fail: ShellClassifierOnFail
|
|
478
|
+
shell_classifier_min_containedness: Unit
|
|
479
|
+
shell_classifier_max_risk: Unit
|
|
480
|
+
shell_classifier_max_modification: Unit
|
|
481
|
+
shell_classifier_max_network: Unit
|
|
443
482
|
sandbox_docker_image: str
|
|
444
483
|
sandbox_timeout_seconds: PositiveInt
|
|
445
484
|
sandbox_memory_mb: PositiveInt
|
|
@@ -655,6 +694,19 @@ DERIVED_VALUES = ("OUTPUT_DIR", "WORKSPACE_DIR_EXPLICIT")
|
|
|
655
694
|
#: authority and cost.
|
|
656
695
|
HARNESS_AUTHORITY_KEYS = frozenset({
|
|
657
696
|
"shell_approval_mode",
|
|
697
|
+
# ROADMAP_v3 §56 (JV8): the decision model's endpoint, model, key name,
|
|
698
|
+
# wait, failure action and thresholds. Each decides whether, or by
|
|
699
|
+
# whom, a shell command is let through unasked.
|
|
700
|
+
"shell_classifier_style",
|
|
701
|
+
"shell_classifier_url",
|
|
702
|
+
"shell_classifier_model",
|
|
703
|
+
"shell_classifier_credential",
|
|
704
|
+
"shell_classifier_timeout_s",
|
|
705
|
+
"shell_classifier_on_fail",
|
|
706
|
+
"shell_classifier_min_containedness",
|
|
707
|
+
"shell_classifier_max_risk",
|
|
708
|
+
"shell_classifier_max_modification",
|
|
709
|
+
"shell_classifier_max_network",
|
|
658
710
|
"allow_insecure_sandbox_fallback",
|
|
659
711
|
"auto_approve_sandbox_fallback",
|
|
660
712
|
"allow_wsl_backend",
|
package/core/commands.py
CHANGED
|
@@ -80,6 +80,10 @@ _SPECS = (
|
|
|
80
80
|
absent={TUI: "ctrl+click a ▸ line to open its call."}),
|
|
81
81
|
CommandSpec("claims", "show a research run's claims and their tiers",
|
|
82
82
|
"[run id]"),
|
|
83
|
+
# ROADMAP_v3 §56 (JV19). Both shells: a line terminal running
|
|
84
|
+
# `classified` needs to see and clear the verdicts as much as the TUI.
|
|
85
|
+
CommandSpec("classifier", "the shell's decision model",
|
|
86
|
+
"[test <command>|clear]"),
|
|
83
87
|
CommandSpec("compact", "summarize this conversation's older turns now",
|
|
84
88
|
"[--strength 1-5]"),
|
|
85
89
|
CommandSpec("config", "browse and set the harness configuration",
|