norn-cli 3.0.0 โ 3.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/NOW.md +70 -20
- package/README.md +74 -2
- package/demos/agent-workbench/norn.config.json +1 -0
- package/demos/mcp-ticket-testing/README.md +114 -0
- package/demos/mcp-ticket-testing/agents.nornagent +77 -0
- package/demos/mcp-ticket-testing/contracts/test-run.schema.json +31 -0
- package/demos/mcp-ticket-testing/expectations/proj-142.md +12 -0
- package/demos/mcp-ticket-testing/fixtures/proj-142.json +13 -0
- package/demos/mcp-ticket-testing/prompts/backend-tester.md +12 -0
- package/demos/mcp-ticket-testing/prompts/frontend-tester.md +14 -0
- package/demos/mcp-ticket-testing/prompts/reporter.md +10 -0
- package/demos/mcp-ticket-testing/servers/browser-server.js +133 -0
- package/demos/mcp-ticket-testing/servers/house-server.js +125 -0
- package/demos/mcp-ticket-testing/tickets.norn +32 -0
- package/dist/cli.js +1324 -459
- package/package.json +3 -3
- package/playground/ai.norn +8 -2
- package/playground/ai_orchastration.nornagent +20 -1
- package/playground/knowedge_base/nexus_system_prompt.md +1 -1
- package/schemas/norn.config.schema.json +12 -0
- package/CHANGELOG.md +0 -1529
package/NOW.md
CHANGED
|
@@ -25,30 +25,28 @@
|
|
|
25
25
|
|
|
26
26
|
## Active now
|
|
27
27
|
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
slice โ it breaks into six items (authoring, rubric/scoring, judge execution, thresholds,
|
|
32
|
-
reporting, fixtures/docs), specced and built one at a time.
|
|
33
|
-
โ Spec: [`Docs/to-be-built/agent-workbench-slice-5.md`](Docs/to-be-built/agent-workbench-slice-5.md)
|
|
34
|
-
|
|
35
|
-
**Next step:** **5f โ fixtures, docs, positioning**, the last item of slice 5. The feature is
|
|
36
|
-
complete and reported across every surface; what is missing is a demo anyone else can run and an
|
|
37
|
-
explanation of what it is for. 5f has no spec yet.
|
|
38
|
-
|
|
39
|
-
**5b and 5d are closed.** 5b was made unnecessary by the checklist scoring model. 5d's content
|
|
40
|
-
was decided in conversation โ a failed judge stops the sequence, the CLI already exits `1`, and
|
|
41
|
-
per-expectation strictness plus the replay-drift check are future work Peter deferred
|
|
42
|
-
2026-08-10.
|
|
43
|
-
โ 5a spec (built): [`Docs/to-be-built/agent-workbench-slice-5a.md`](Docs/to-be-built/agent-workbench-slice-5a.md)
|
|
44
|
-
โ 5c spec (built): [`Docs/to-be-built/agent-workbench-slice-5c.md`](Docs/to-be-built/agent-workbench-slice-5c.md)
|
|
45
|
-
โ 5e spec (built): [`Docs/to-be-built/agent-workbench-slice-5e.md`](Docs/to-be-built/agent-workbench-slice-5e.md)
|
|
28
|
+
_Nothing accepted right now._ The MCP-servers item finished 2026-08-12 โ see **Recently done**.
|
|
29
|
+
Slice 5f is the standing follow-on in the backlog below; it now has a spec and is ready for a
|
|
30
|
+
fresh session the moment Peter accepts it here.
|
|
46
31
|
|
|
47
32
|
---
|
|
48
33
|
|
|
49
34
|
## Backlog / accepted-but-not-started
|
|
50
35
|
|
|
51
|
-
|
|
36
|
+
### ๐งช Agent Workbench โ Slice 5f: eval fixtures, docs, positioning
|
|
37
|
+
**Postponed 2026-08-11; the MCP work it was waiting on finished 2026-08-12, and it now has a
|
|
38
|
+
spec.** Slice 5 is otherwise complete and reported across every surface (5a, 5c, 5e built; 5b and
|
|
39
|
+
5d closed). 5f is smaller than its name suggests: the **website already documents `judge` well**,
|
|
40
|
+
and a deterministic judge server already exists as a test fixture โ so what is left is promoting
|
|
41
|
+
that fixture to a demo anyone can run with **no API key** (one clearly-good case, one clearly-bad
|
|
42
|
+
one), a README section that catches up with the website, and the explanation of what evals are
|
|
43
|
+
for. **No `src/` change.** The spec leads with the constraint that matters: Norn's positioning is
|
|
44
|
+
locked, so "positioning" here means explaining evals *from* the spine, never a new pitch.
|
|
45
|
+
โ **5f spec: [`Docs/to-be-built/agent-workbench-slice-5f.md`](Docs/to-be-built/agent-workbench-slice-5f.md)**
|
|
46
|
+
โ Slice 5 spec: [`Docs/to-be-built/agent-workbench-slice-5.md`](Docs/to-be-built/agent-workbench-slice-5.md)
|
|
47
|
+
โ Built: [`5a`](Docs/to-be-built/agent-workbench-slice-5a.md) ยท
|
|
48
|
+
[`5c`](Docs/to-be-built/agent-workbench-slice-5c.md) ยท
|
|
49
|
+
[`5e`](Docs/to-be-built/agent-workbench-slice-5e.md)
|
|
52
50
|
|
|
53
51
|
## Parked
|
|
54
52
|
|
|
@@ -60,6 +58,58 @@ narrative and how to resume:
|
|
|
60
58
|
|
|
61
59
|
## Recently done
|
|
62
60
|
|
|
61
|
+
- **3.1.0 โ MCP authoring parity, and the changelog is gone** (2026-08-13) โ **done.**
|
|
62
|
+
Three fixes that all came out of one painful hour authoring a real server by hand. **`header`
|
|
63
|
+
now reads like HTTP**: `header Name: value` is accepted alongside the original no-colon form,
|
|
64
|
+
because the colon form is what a `.norn` request writes and what everyone arrives with โ
|
|
65
|
+
rejecting it silently produced a header *named* `Authorization:` and the only complaint came
|
|
66
|
+
from `Headers.append` at dial time, three layers from the line at fault. The name is now
|
|
67
|
+
validated against the RFC 9110 token set at parse time, so an unsendable one is red in the
|
|
68
|
+
editor. The header name and value carry `.norn`'s own `entity.name.tag.http` /
|
|
69
|
+
`string.unquoted.http` scopes, so the identical header is the identical colour in both files,
|
|
70
|
+
and the quoted-string pattern is gone from that rule โ it advertised an escape that does not
|
|
71
|
+
exist, and quotes really do end up inside the token. IntelliSense offers the shared
|
|
72
|
+
`commonHeaders` list after `header `, inserting the colon form and chaining into the
|
|
73
|
+
`Bearer {{$env.โฆ}}` value completion.
|
|
74
|
+
**`run mcp` completions now see sidecar-declared servers.** The precedence rule lived in three
|
|
75
|
+
copies and the completion copy was left on config-only, so a project that declared every server
|
|
76
|
+
in its sidecar offered no aliases at all while running perfectly. It is now one shared module,
|
|
77
|
+
`src/mcpAliasScope.ts`, and the alias carries its origin โ which matters because the tools cache
|
|
78
|
+
is written beside whatever *declared* the server, so tool-name completions and signature help
|
|
79
|
+
were reading the wrong `.norn-cache` even once an alias resolved. **Config `mcp.servers` stays**:
|
|
80
|
+
supporting both is one function and one line per caller, so retiring it would have been strictly
|
|
81
|
+
more work than keeping it.
|
|
82
|
+
**`CHANGELOG.md` deleted** at Peter's call โ the release record is git history plus this board,
|
|
83
|
+
and a changelog was a third place saying the same thing and the likeliest to drift.
|
|
84
|
+
Also fixed: `@vscode/test-electron` 2.5.2 could not launch VS Code 1.133 (the binary was renamed
|
|
85
|
+
from `Electron` to `Code`), which had left the Extension Host suite dark.
|
|
86
|
+
- **Agent Workbench โ declared MCP servers** (2026-08-12) โ **done, all five items.**
|
|
87
|
+
Agents can now *do* things. A `.nornagent` sidecar declares the servers it uses beside the
|
|
88
|
+
models it uses โ `mcp <Alias> โฆ end mcp`, stdio and http, credentials as `.nornenv`
|
|
89
|
+
references โ and each agent is granted a whole server with `mcp <Alias>` or named tools with
|
|
90
|
+
`tools <Alias>.<tool>`. The two stay visibly different statements on purpose: a granted tool
|
|
91
|
+
executes when the model asks, so the authored grant *is* the permission boundary and dropping
|
|
92
|
+
four characters must not widen it from one tool to twenty. `session run | agent | call` makes
|
|
93
|
+
shared server state a decision rather than a surprise; `run` stays the default because five
|
|
94
|
+
agents testing one website are one testing session, and `session agent` is how an agent asks
|
|
95
|
+
not to inherit a login it never earned. Config `mcp.servers` keeps working with the sidecar
|
|
96
|
+
winning, `run mcp` steps reach declared servers so a server is provable with no model and no
|
|
97
|
+
cost, and a granted server is a real node on the canvas โ **declared servers only**, per
|
|
98
|
+
Peter: the graph draws the program the file describes.
|
|
99
|
+
Four corrections the build forced on the spec, all recorded in it: grant expansion **cannot**
|
|
100
|
+
live above the runner (pure replay never runs that preflight, and a derived root definition
|
|
101
|
+
never reaches sub-agents), so it is an injected hook with a live and a replay implementation;
|
|
102
|
+
drift between record and replay is only knowable where a live listing happens, so the
|
|
103
|
+
acceptance criterion demanding both was impossible as written; preflight needed a session
|
|
104
|
+
boundary of its own, or `session agent` was already broken by the check that proves an agent's
|
|
105
|
+
tools exist; and the spec's own demo sketch used an input expression the language does not
|
|
106
|
+
have. Fixed along the way: a templated URL resolving differently mid-run silently reused the
|
|
107
|
+
old session, and a refused connection reported the SDK's "fetch failed" instead of naming what
|
|
108
|
+
it dialled. 60 focused tests across five items; 256 pass across every runnable suite, plus
|
|
109
|
+
type-check, lint, bundles, skill validation, and regression at 198/200 (two live-SQL-Server
|
|
110
|
+
sequences needing a database).
|
|
111
|
+
โ Spec: [`Docs/to-be-built/agent-workbench-mcp-servers.md`](Docs/to-be-built/agent-workbench-mcp-servers.md)
|
|
112
|
+
ยท Demo: [`demos/mcp-ticket-testing`](demos/mcp-ticket-testing)
|
|
63
113
|
- **Agent Workbench Slice 5e โ reporting** (2026-08-10) โ **done.**
|
|
64
114
|
A verdict is now visible in the response panel, the canvas, the debugger, and JUnit, derived
|
|
65
115
|
once through `agentHopPresentation` so the surfaces cannot disagree. The load-bearing move was
|
|
@@ -154,4 +204,4 @@ narrative and how to resume:
|
|
|
154
204
|
and functionality untouched. Was Phase 0 of the marketing plan.
|
|
155
205
|
|
|
156
206
|
---
|
|
157
|
-
_Last updated: 2026-08-
|
|
207
|
+
_Last updated: 2026-08-12_
|
package/README.md
CHANGED
|
@@ -107,7 +107,14 @@ Every `{{...}}` reference in any Norn file shows its resolved value as a gray in
|
|
|
107
107
|
|
|
108
108
|
## Deterministic MCP Tools
|
|
109
109
|
|
|
110
|
-
Norn can call MCP tools from sequences without leaving the `.norn` runtime. MCP sessions are deterministic and shared across the full sequence run, so nested sequences reuse the same connection for the same resolved server alias.
|
|
110
|
+
Norn can call MCP tools from sequences without leaving the `.norn` runtime. MCP sessions are deterministic and shared across the full sequence run by default, so nested sequences reuse the same connection for the same resolved server alias.
|
|
111
|
+
|
|
112
|
+
A server can be declared in two places, and an alias resolves through them in this order:
|
|
113
|
+
|
|
114
|
+
1. an `mcp <Alias> ... end mcp` block in a `.nornagent` sidecar the file imports โ see
|
|
115
|
+
[Contract-Checked Agent Graphs](#contract-checked-agent-graphs), and prefer this when agents
|
|
116
|
+
use the server too, because the file then says what it connects to;
|
|
117
|
+
2. `mcp.servers` in the nearest `norn.config.json`.
|
|
111
118
|
|
|
112
119
|
Create a `norn.config.json` in the root of your project:
|
|
113
120
|
|
|
@@ -150,7 +157,11 @@ Behavior:
|
|
|
150
157
|
- `run mcp list <alias>` returns the full tool list and drains paginated `nextCursor` responses automatically.
|
|
151
158
|
- `run mcp call <alias> <tool>(...)` supports named arguments or positional arguments bound in tool-schema order, and returns a deterministic result envelope with `content`, `structuredContent`, `isError`, `text`, `server`, and `tool`.
|
|
152
159
|
- Tool `structuredContent` is validated against the MCP tool's advertised `outputSchema` when present.
|
|
153
|
-
-
|
|
160
|
+
- A run-scoped session is closed automatically when the outermost sequence finishes or fails. A
|
|
161
|
+
sidecar-declared server can narrow that with `session agent` or `session call`, which close at
|
|
162
|
+
their own boundary instead.
|
|
163
|
+
- An imported `.norn` helper sequence resolves aliases through **its own** imports, not its
|
|
164
|
+
caller's โ the same rule agent scopes follow.
|
|
154
165
|
|
|
155
166
|
## Contract-Checked Agent Graphs
|
|
156
167
|
|
|
@@ -188,6 +199,67 @@ agent TicketRouter
|
|
|
188
199
|
end agent
|
|
189
200
|
```
|
|
190
201
|
|
|
202
|
+
### Giving agents hands: declared MCP servers
|
|
203
|
+
|
|
204
|
+
An agent reaches the outside world through MCP servers, and those are declared in the same
|
|
205
|
+
file, the same way โ every input the server needs written down, credentials as `.nornenv`
|
|
206
|
+
references:
|
|
207
|
+
|
|
208
|
+
```nornagent
|
|
209
|
+
mcp Browser
|
|
210
|
+
transport stdio
|
|
211
|
+
command npx playwright-mcp --headless --isolated
|
|
212
|
+
session agent # run (default) | agent | call
|
|
213
|
+
end mcp
|
|
214
|
+
|
|
215
|
+
mcp House
|
|
216
|
+
transport http
|
|
217
|
+
url {{$env.HOUSE_MCP_URL}} # the MCP endpoint, path included
|
|
218
|
+
header Authorization: Bearer {{$env.HOUSE_TOKEN}}
|
|
219
|
+
timeout 60000
|
|
220
|
+
end mcp
|
|
221
|
+
|
|
222
|
+
agent FrontendTester
|
|
223
|
+
model Workbench
|
|
224
|
+
mcp Browser # every tool this server advertises
|
|
225
|
+
tools House.getFixtureUser, House.resetTenant # only these, from this one
|
|
226
|
+
system file prompts/frontend-tester.md
|
|
227
|
+
end agent
|
|
228
|
+
```
|
|
229
|
+
|
|
230
|
+
- **`transport stdio` spawns the server per run; `transport http` only dials one.** Norn never
|
|
231
|
+
starts an http server, so it must already be listening โ a refused connection fails before
|
|
232
|
+
any model spend, naming the URL it tried.
|
|
233
|
+
- **`mcp <Alias>` and `tools <Alias>.<tool>` are deliberately different statements.** A granted
|
|
234
|
+
tool executes when the model asks for it, so the authored grant *is* the permission boundary:
|
|
235
|
+
dropping four characters must not silently widen access from one tool to twenty.
|
|
236
|
+
- **`header` writes an HTTP header the way you would write it anywhere else.** `header Name:
|
|
237
|
+
value`, and the value is the rest of the line taken verbatim โ so `Bearer {{token}}` needs no
|
|
238
|
+
quotes, and quotes you add become part of the token. The colon is optional but recommended:
|
|
239
|
+
it is what the request side of Norn uses, and IntelliSense inserts it for you.
|
|
240
|
+
- **`session` decides what state is shared.** The default, `run`, gives every agent in a
|
|
241
|
+
sequence one session โ deliberate for an orchestration, and a trap when a backend agent
|
|
242
|
+
inherits a frontend agent's login and passes a test that should have failed. `session agent`
|
|
243
|
+
is how an agent asks for its own.
|
|
244
|
+
- An alias with no block still resolves through `norn.config.json` `mcp.servers`, and a block
|
|
245
|
+
in the sidecar wins over a config entry of the same name.
|
|
246
|
+
|
|
247
|
+
Deterministic `run mcp` steps resolve the same aliases, so a server can be proved reachable
|
|
248
|
+
with no model and no cost:
|
|
249
|
+
|
|
250
|
+
```norn
|
|
251
|
+
import "./agents.nornagent"
|
|
252
|
+
|
|
253
|
+
test sequence ServersUp
|
|
254
|
+
var tools = run mcp list House
|
|
255
|
+
assert tools.length > 0
|
|
256
|
+
end sequence
|
|
257
|
+
```
|
|
258
|
+
|
|
259
|
+
The runnable [`demos/mcp-ticket-testing`](./demos/mcp-ticket-testing) example is the whole
|
|
260
|
+
shape end to end: a browser agent, a backend agent narrowed on the same custom server, a
|
|
261
|
+
reporter, and a `judge` over the expected test cases.
|
|
262
|
+
|
|
191
263
|
A prompt that outgrows its sidecar can live in its own file instead. `describe`
|
|
192
264
|
and `system` both accept `file <path>`, resolved relative to the `.nornagent`
|
|
193
265
|
file just like an import:
|
|
@@ -0,0 +1,114 @@
|
|
|
1
|
+
# Ticket testing with declared MCP servers
|
|
2
|
+
|
|
3
|
+
An agent graph that **does** something: a ticket arrives, one agent tests the product through a
|
|
4
|
+
browser, another tests it through the team's own API, a third writes the results up, and a
|
|
5
|
+
`judge` rules on whether the expected test cases were actually covered.
|
|
6
|
+
|
|
7
|
+
Everything it connects to is declared in [`agents.nornagent`](./agents.nornagent), beside the
|
|
8
|
+
model. There is **no `norn.config.json` in this folder** โ the servers, their transports, and
|
|
9
|
+
their session lifetimes are all in the file that uses them.
|
|
10
|
+
|
|
11
|
+
## What to look at
|
|
12
|
+
|
|
13
|
+
```nornagent
|
|
14
|
+
mcp Browser
|
|
15
|
+
transport stdio
|
|
16
|
+
command node ./servers/browser-server.js
|
|
17
|
+
session agent
|
|
18
|
+
end mcp
|
|
19
|
+
|
|
20
|
+
agent FrontendTester
|
|
21
|
+
mcp Browser # the run of the browser
|
|
22
|
+
tools House.getFixtureUser, House.resetTenant # only what it needs from ours
|
|
23
|
+
end agent
|
|
24
|
+
|
|
25
|
+
agent BackendTester
|
|
26
|
+
tools House.callEndpoint, House.readAuditLog, House.queryDb
|
|
27
|
+
end agent
|
|
28
|
+
```
|
|
29
|
+
|
|
30
|
+
Three things this shows, and they are the point of the whole slice:
|
|
31
|
+
|
|
32
|
+
- **Two agents, one custom server, different tools each.** `BackendTester` gets three named
|
|
33
|
+
tools from `House` and cannot touch the browser at all. The access boundary is visible per
|
|
34
|
+
agent, in one file, in git.
|
|
35
|
+
- **`mcp Browser` and `tools House.callEndpoint` are visibly different statements.** Dropping
|
|
36
|
+
four characters must not silently widen access from one tool to twenty, so there is no
|
|
37
|
+
short form that means "all of it".
|
|
38
|
+
- **`session agent` on the browser.** Each agent invocation gets its own browser, closed when
|
|
39
|
+
that hop ends. The default, `session run`, shares one browser across every agent in the
|
|
40
|
+
sequence โ right for a continuous flow, and wrong the moment a second agent is granted the
|
|
41
|
+
same server: it would inherit the first one's login and could pass a test it never
|
|
42
|
+
authenticated for, which is a false green and worse than a false red. Only `FrontendTester`
|
|
43
|
+
reaches the browser in this graph, so this is the lifetime worth authoring by default rather
|
|
44
|
+
than a bleed this demo would actually suffer.
|
|
45
|
+
|
|
46
|
+
## Run it
|
|
47
|
+
|
|
48
|
+
The servers are deterministic and ship with the demo, so nothing needs installing or starting.
|
|
49
|
+
Prove they are reachable **without spending anything on a model**:
|
|
50
|
+
|
|
51
|
+
```bash
|
|
52
|
+
npm run compile
|
|
53
|
+
node ./dist/cli.js demos/mcp-ticket-testing/tickets.norn --sequence ServersUp
|
|
54
|
+
```
|
|
55
|
+
|
|
56
|
+
That resolves the same aliases the agents use, so a failure points at the server rather than at
|
|
57
|
+
an agent. When you are building a server of your own, this is the loop to develop against.
|
|
58
|
+
|
|
59
|
+
Then the real thing, which does need a model:
|
|
60
|
+
|
|
61
|
+
```bash
|
|
62
|
+
export OPENAI_API="your-key"
|
|
63
|
+
node ./dist/cli.js demos/mcp-ticket-testing/tickets.norn --sequence TicketPROJ142
|
|
64
|
+
```
|
|
65
|
+
|
|
66
|
+
The variable name is not a convention Norn assumes โ it is written in the model block as
|
|
67
|
+
`apiKey {{$env.OPENAI_API}}`, so any name works as long as the file and the environment
|
|
68
|
+
agree. A `.nornenv` value wins over the process environment; add `--env <name>` to select one.
|
|
69
|
+
|
|
70
|
+
## The browser here is a stand-in
|
|
71
|
+
|
|
72
|
+
[`servers/browser-server.js`](./servers/browser-server.js) is a deterministic scripted site, not
|
|
73
|
+
a real browser, so this demo runs anywhere `node` does โ no browser download, no server to keep
|
|
74
|
+
alive. Its declaration is the **same shape** a real Playwright MCP block takes, and both real
|
|
75
|
+
forms are written out in the comments at the top of
|
|
76
|
+
[`agents.nornagent`](./agents.nornagent):
|
|
77
|
+
|
|
78
|
+
```nornagent
|
|
79
|
+
mcp Browser
|
|
80
|
+
transport stdio
|
|
81
|
+
command npx playwright-mcp --headless --isolated
|
|
82
|
+
session agent
|
|
83
|
+
end mcp
|
|
84
|
+
```
|
|
85
|
+
|
|
86
|
+
For CI with more than one sequence, run Playwright MCP as a service and dial it instead โ that
|
|
87
|
+
is what `transport http` is for. Norn never starts an http server; it only connects to one, and
|
|
88
|
+
a refused connection fails before any model spend, naming the URL it tried.
|
|
89
|
+
|
|
90
|
+
## The judge, and the CI story
|
|
91
|
+
|
|
92
|
+
The last line of `TicketPROJ142` is the part that makes this a test rather than a demo:
|
|
93
|
+
|
|
94
|
+
```norn
|
|
95
|
+
judge report with Reviewer expects file expectations/proj-142.md
|
|
96
|
+
```
|
|
97
|
+
|
|
98
|
+
Every expectation in [`expectations/proj-142.md`](./expectations/proj-142.md) must be met;
|
|
99
|
+
testing *more* than the list is not a failure. The judge is never asked for an overall verdict โ
|
|
100
|
+
it rules on each expectation and quotes its evidence, and Norn computes the result.
|
|
101
|
+
|
|
102
|
+
Note the comment lines in that file. A line of prose left uncommented there **is** an
|
|
103
|
+
expectation, which is both Norn's comment rule and what makes Markdown headings work.
|
|
104
|
+
|
|
105
|
+
A live run records itself under `.norn-cache/runs/`. Replay it with no servers running at all:
|
|
106
|
+
|
|
107
|
+
```bash
|
|
108
|
+
node ./dist/cli.js replay .norn-cache/runs/<recording>.json
|
|
109
|
+
```
|
|
110
|
+
|
|
111
|
+
That is the CI answer for any expensive server: run it live where the servers exist, and gate
|
|
112
|
+
pull requests on the replay โ no browser, no API, no model, and the graph and contracts still
|
|
113
|
+
checked. The recording keeps the tool names each granted server advertised, so a server that
|
|
114
|
+
quietly gains or loses a tool is reported the next time a live run sees it.
|
|
@@ -0,0 +1,77 @@
|
|
|
1
|
+
# Ticket testing โ declared MCP servers, granted per agent.
|
|
2
|
+
#
|
|
3
|
+
# Every server this graph uses is declared here, beside the model, so the file says exactly
|
|
4
|
+
# what it connects to. Nothing is left to a norn.config.json the reader cannot see.
|
|
5
|
+
|
|
6
|
+
model Workbench
|
|
7
|
+
provider openai
|
|
8
|
+
name gpt-4o
|
|
9
|
+
apiKey {{$env.OPENAI_API}}
|
|
10
|
+
end model
|
|
11
|
+
|
|
12
|
+
# The browser. This demo ships a deterministic stand-in so it runs anywhere `node` does โ
|
|
13
|
+
# the declaration is the same shape a real Playwright MCP server takes:
|
|
14
|
+
#
|
|
15
|
+
# mcp Browser
|
|
16
|
+
# transport stdio
|
|
17
|
+
# command npx playwright-mcp --headless --isolated
|
|
18
|
+
# session agent
|
|
19
|
+
# end mcp
|
|
20
|
+
#
|
|
21
|
+
# ...or, for a long-lived server you started yourself (a CI job with many sequences):
|
|
22
|
+
#
|
|
23
|
+
# mcp Browser
|
|
24
|
+
# transport http
|
|
25
|
+
# url {{$env.BROWSER_MCP_URL}} # e.g. http://localhost:8931/mcp
|
|
26
|
+
# timeout 120000 # browser work is slow; the default is not for this
|
|
27
|
+
# end mcp
|
|
28
|
+
mcp Browser
|
|
29
|
+
transport stdio
|
|
30
|
+
command node ./servers/browser-server.js
|
|
31
|
+
# One browser per agent invocation, closed when that hop ends. The default, `session run`,
|
|
32
|
+
# would share one browser across every agent in the sequence โ which is what you want for a
|
|
33
|
+
# continuous flow, and what you do not want the moment a second agent is granted this server:
|
|
34
|
+
# it would inherit the first one's login and could pass a test it never authenticated for.
|
|
35
|
+
# Only FrontendTester reaches the browser here, so this is the safe default to author rather
|
|
36
|
+
# than a bleed this graph would actually suffer.
|
|
37
|
+
session agent
|
|
38
|
+
end mcp
|
|
39
|
+
|
|
40
|
+
# The team's own server: fixtures, endpoints, the audit log, the database.
|
|
41
|
+
mcp House
|
|
42
|
+
transport stdio
|
|
43
|
+
command node ./servers/house-server.js
|
|
44
|
+
end mcp
|
|
45
|
+
|
|
46
|
+
agent FrontendTester
|
|
47
|
+
model Workbench
|
|
48
|
+
# The run of the browser: this agent drives the site, and which tools that needs is not
|
|
49
|
+
# something the author should have to enumerate.
|
|
50
|
+
mcp Browser
|
|
51
|
+
# From our own server it gets only what it needs to set up โ and cannot reach the
|
|
52
|
+
# database or the audit log at all.
|
|
53
|
+
tools House.getFixtureUser, House.resetTenant
|
|
54
|
+
returns contracts/test-run.schema.json
|
|
55
|
+
system file prompts/frontend-tester.md
|
|
56
|
+
end agent
|
|
57
|
+
|
|
58
|
+
agent BackendTester
|
|
59
|
+
model Workbench
|
|
60
|
+
# The same server, narrowed differently. Both grants are visible per agent, in one file.
|
|
61
|
+
tools House.callEndpoint, House.readAuditLog, House.queryDb
|
|
62
|
+
returns contracts/test-run.schema.json
|
|
63
|
+
system file prompts/backend-tester.md
|
|
64
|
+
end agent
|
|
65
|
+
|
|
66
|
+
agent Reporter
|
|
67
|
+
model Workbench
|
|
68
|
+
system file prompts/reporter.md
|
|
69
|
+
end agent
|
|
70
|
+
|
|
71
|
+
agent Reviewer
|
|
72
|
+
model Workbench
|
|
73
|
+
system "
|
|
74
|
+
You are reviewing a QA report against a checklist of expected test cases.
|
|
75
|
+
Rule on each expectation independently and quote the report as evidence.
|
|
76
|
+
"
|
|
77
|
+
end agent
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
{
|
|
2
|
+
"type": "object",
|
|
3
|
+
"additionalProperties": false,
|
|
4
|
+
"properties": {
|
|
5
|
+
"surface": {
|
|
6
|
+
"type": "string",
|
|
7
|
+
"description": "Which side of the product was exercised: frontend or backend."
|
|
8
|
+
},
|
|
9
|
+
"cases": {
|
|
10
|
+
"type": "array",
|
|
11
|
+
"description": "One entry per test case actually carried out.",
|
|
12
|
+
"items": {
|
|
13
|
+
"type": "object",
|
|
14
|
+
"additionalProperties": false,
|
|
15
|
+
"properties": {
|
|
16
|
+
"name": { "type": "string" },
|
|
17
|
+
"steps": { "type": "array", "items": { "type": "string" } },
|
|
18
|
+
"outcome": { "type": "string", "enum": ["passed", "failed"] },
|
|
19
|
+
"evidence": { "type": "string" }
|
|
20
|
+
},
|
|
21
|
+
"required": ["name", "steps", "outcome", "evidence"]
|
|
22
|
+
}
|
|
23
|
+
},
|
|
24
|
+
"findings": {
|
|
25
|
+
"type": "array",
|
|
26
|
+
"description": "Anything that looked wrong. Empty when nothing did.",
|
|
27
|
+
"items": { "type": "string" }
|
|
28
|
+
}
|
|
29
|
+
},
|
|
30
|
+
"required": ["surface", "cases", "findings"]
|
|
31
|
+
}
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
# Expected test cases for PROJ-142
|
|
2
|
+
|
|
3
|
+
# Each line below is one expectation. All must be met; testing more than this is not a
|
|
4
|
+
# failure. Lines starting with # are comments โ prose left uncommented here would itself
|
|
5
|
+
# become an expectation, which is both Norn's comment rule and what makes Markdown work.
|
|
6
|
+
|
|
7
|
+
- A test covers that a signed-out visitor opening /orders is sent to the sign-in page
|
|
8
|
+
- A test covers signing in and seeing the orders list
|
|
9
|
+
- A test covers opening order A-1001 and reading its total
|
|
10
|
+
- A test covers GET /orders/A-1001 returning 200 with the order
|
|
11
|
+
- A test covers GET /orders/A-9999 returning 404
|
|
12
|
+
- A test confirms endpoint calls reach the audit log
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
{
|
|
2
|
+
"key": "PROJ-142",
|
|
3
|
+
"title": "Order detail page and order lookup endpoint",
|
|
4
|
+
"description": "A signed-in customer can open an order from the orders list and see its total. The API must return the order for a valid id and 404 for an unknown one.",
|
|
5
|
+
"acceptanceCriteria": [
|
|
6
|
+
"A signed-out visitor who opens /orders is sent to the sign-in page",
|
|
7
|
+
"After signing in, the orders list shows the customer's orders",
|
|
8
|
+
"Opening order A-1001 shows its total",
|
|
9
|
+
"GET /orders/A-1001 returns 200 with the order",
|
|
10
|
+
"GET /orders/A-9999 returns 404",
|
|
11
|
+
"Every endpoint call is written to the audit log"
|
|
12
|
+
]
|
|
13
|
+
}
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
You test the backend of a web product through its team's own MCP server.
|
|
2
|
+
|
|
3
|
+
Work only from the ticket you are given. For each acceptance criterion that is about an API,
|
|
4
|
+
a database row, or an audit entry, exercise it with the tools you have and record what came
|
|
5
|
+
back โ never what you assume would come back.
|
|
6
|
+
|
|
7
|
+
`callEndpoint` calls the API. `queryDb` reads seeded tables. `readAuditLog` returns what this
|
|
8
|
+
run wrote. You have no browser, and you must not claim to have used one.
|
|
9
|
+
|
|
10
|
+
Return the required JSON only: one case per criterion you exercised, the steps you took, the
|
|
11
|
+
outcome, and a short quote of the evidence you saw. Put anything that looked wrong in
|
|
12
|
+
`findings`. Set `surface` to "backend".
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
You test the front end of a web product by driving a browser through an MCP server.
|
|
2
|
+
|
|
3
|
+
Work only from the ticket you are given. For each acceptance criterion that is about what a
|
|
4
|
+
person sees or does in the browser, carry out the steps yourself with the browser tools and
|
|
5
|
+
record what actually happened โ never what you assume would happen.
|
|
6
|
+
|
|
7
|
+
The browser starts signed out. `getFixtureUser` gives you the test account; `resetTenant`
|
|
8
|
+
puts the tenant back to its seeded state before you start.
|
|
9
|
+
|
|
10
|
+
Snapshots are text. Read the elements listed in a snapshot rather than guessing at a page.
|
|
11
|
+
|
|
12
|
+
Return the required JSON only: one case per criterion you exercised, the steps you took, the
|
|
13
|
+
outcome, and a short quote of the evidence you saw. Put anything that looked wrong in
|
|
14
|
+
`findings`. Set `surface` to "frontend".
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
You turn two test runs into one short QA report for a ticket.
|
|
2
|
+
|
|
3
|
+
You are given the frontend run and the backend run. Write, in plain prose:
|
|
4
|
+
|
|
5
|
+
1. Which test cases were carried out, naming each one.
|
|
6
|
+
2. What the evidence was for each.
|
|
7
|
+
3. Any findings, or a single sentence saying there were none.
|
|
8
|
+
|
|
9
|
+
Report only what the runs contain. Do not invent a case that nobody ran, and do not soften a
|
|
10
|
+
failure. Someone will check this report against the list of expected test cases.
|
|
@@ -0,0 +1,133 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* A deterministic stand-in for Playwright MCP.
|
|
5
|
+
*
|
|
6
|
+
* The `mcp Browser` block in `agents.nornagent` is the *same shape* a real Playwright MCP
|
|
7
|
+
* declaration takes โ see the commented block there โ so this demo runs anywhere `node` does,
|
|
8
|
+
* with no browser download, while showing exactly what granting a browser looks like.
|
|
9
|
+
*
|
|
10
|
+
* It answers from a tiny scripted site, and its snapshots are text, which is what a browser
|
|
11
|
+
* agent should be working from anyway: image content never reaches a model's prompt.
|
|
12
|
+
*/
|
|
13
|
+
|
|
14
|
+
const { Server } = require('@modelcontextprotocol/sdk/server/index.js');
|
|
15
|
+
const { StdioServerTransport } = require('@modelcontextprotocol/sdk/server/stdio.js');
|
|
16
|
+
const { ListToolsRequestSchema, CallToolRequestSchema } = require('@modelcontextprotocol/sdk/types.js');
|
|
17
|
+
|
|
18
|
+
const PAGES = {
|
|
19
|
+
'/login': {
|
|
20
|
+
title: 'Sign in',
|
|
21
|
+
elements: ['textbox "Email"', 'textbox "Password"', 'button "Sign in"']
|
|
22
|
+
},
|
|
23
|
+
'/orders': {
|
|
24
|
+
title: 'Your orders',
|
|
25
|
+
elements: ['link "A-1001"', 'link "A-1002"', 'button "New order"']
|
|
26
|
+
},
|
|
27
|
+
'/orders/A-1001': {
|
|
28
|
+
title: 'Order A-1001',
|
|
29
|
+
elements: ['heading "Order A-1001"', 'text "Total: 42.50"', 'button "Cancel order"']
|
|
30
|
+
}
|
|
31
|
+
};
|
|
32
|
+
|
|
33
|
+
const state = { url: '/login', signedIn: false, filled: {} };
|
|
34
|
+
|
|
35
|
+
const TOOLS = [
|
|
36
|
+
{
|
|
37
|
+
name: 'browser_navigate',
|
|
38
|
+
description: 'Navigate to a path on the site under test.',
|
|
39
|
+
inputSchema: {
|
|
40
|
+
type: 'object',
|
|
41
|
+
additionalProperties: false,
|
|
42
|
+
properties: { path: { type: 'string', description: 'Path to open, e.g. /orders.' } },
|
|
43
|
+
required: ['path']
|
|
44
|
+
}
|
|
45
|
+
},
|
|
46
|
+
{
|
|
47
|
+
name: 'browser_fill',
|
|
48
|
+
description: 'Fill a named field on the current page.',
|
|
49
|
+
inputSchema: {
|
|
50
|
+
type: 'object',
|
|
51
|
+
additionalProperties: false,
|
|
52
|
+
properties: {
|
|
53
|
+
field: { type: 'string', description: 'Field label, e.g. Email.' },
|
|
54
|
+
value: { type: 'string', description: 'Value to type.' }
|
|
55
|
+
},
|
|
56
|
+
required: ['field', 'value']
|
|
57
|
+
}
|
|
58
|
+
},
|
|
59
|
+
{
|
|
60
|
+
name: 'browser_click',
|
|
61
|
+
description: 'Click a named control on the current page.',
|
|
62
|
+
inputSchema: {
|
|
63
|
+
type: 'object',
|
|
64
|
+
additionalProperties: false,
|
|
65
|
+
properties: { control: { type: 'string', description: 'Control label, e.g. Sign in.' } },
|
|
66
|
+
required: ['control']
|
|
67
|
+
}
|
|
68
|
+
},
|
|
69
|
+
{
|
|
70
|
+
name: 'browser_snapshot',
|
|
71
|
+
description: 'Return a text accessibility snapshot of the current page.',
|
|
72
|
+
inputSchema: { type: 'object', additionalProperties: false, properties: {} }
|
|
73
|
+
}
|
|
74
|
+
];
|
|
75
|
+
|
|
76
|
+
function structured(value) {
|
|
77
|
+
return { content: [{ type: 'text', text: JSON.stringify(value) }], structuredContent: value };
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
function snapshot() {
|
|
81
|
+
const page = PAGES[state.url] || { title: 'Not found', elements: [] };
|
|
82
|
+
return {
|
|
83
|
+
url: state.url,
|
|
84
|
+
title: page.title,
|
|
85
|
+
signedIn: state.signedIn,
|
|
86
|
+
elements: page.elements
|
|
87
|
+
};
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
async function main() {
|
|
91
|
+
const server = new Server(
|
|
92
|
+
{ name: 'norn-demo-browser', version: '1.0.0' },
|
|
93
|
+
{ capabilities: { tools: {} } }
|
|
94
|
+
);
|
|
95
|
+
|
|
96
|
+
server.setRequestHandler(ListToolsRequestSchema, async () => ({ tools: TOOLS }));
|
|
97
|
+
|
|
98
|
+
server.setRequestHandler(CallToolRequestSchema, async request => {
|
|
99
|
+
const args = request.params.arguments || {};
|
|
100
|
+
switch (request.params.name) {
|
|
101
|
+
case 'browser_navigate':
|
|
102
|
+
state.url = String(args.path || '/');
|
|
103
|
+
if (state.url !== '/login' && !state.signedIn) {
|
|
104
|
+
// The whole point of a session: an unauthenticated visit lands on login.
|
|
105
|
+
state.url = '/login';
|
|
106
|
+
}
|
|
107
|
+
return structured(snapshot());
|
|
108
|
+
case 'browser_fill':
|
|
109
|
+
state.filled[String(args.field)] = String(args.value);
|
|
110
|
+
return structured({ filled: args.field, ...snapshot() });
|
|
111
|
+
case 'browser_click':
|
|
112
|
+
if (String(args.control) === 'Sign in' && state.filled.Email && state.filled.Password) {
|
|
113
|
+
state.signedIn = true;
|
|
114
|
+
state.url = '/orders';
|
|
115
|
+
}
|
|
116
|
+
return structured(snapshot());
|
|
117
|
+
case 'browser_snapshot':
|
|
118
|
+
return structured(snapshot());
|
|
119
|
+
default:
|
|
120
|
+
return {
|
|
121
|
+
isError: true,
|
|
122
|
+
content: [{ type: 'text', text: `Unknown tool: ${request.params.name}` }]
|
|
123
|
+
};
|
|
124
|
+
}
|
|
125
|
+
});
|
|
126
|
+
|
|
127
|
+
await server.connect(new StdioServerTransport());
|
|
128
|
+
}
|
|
129
|
+
|
|
130
|
+
main().catch(error => {
|
|
131
|
+
console.error(error instanceof Error ? error.message : String(error));
|
|
132
|
+
process.exit(1);
|
|
133
|
+
});
|