@elench/testkit 0.1.93 → 0.1.95
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +20 -7
- package/lib/cli/agents/providers/claude.mjs +28 -5
- package/lib/cli/agents/providers/codex.mjs +30 -11
- package/lib/cli/agents/providers/shared.mjs +5 -0
- package/lib/cli/assistant/app.mjs +112 -83
- package/lib/cli/assistant/context-pack.mjs +5 -4
- package/lib/cli/assistant/context-window.mjs +69 -0
- package/lib/cli/assistant/session.mjs +7 -0
- package/lib/cli/assistant/state.mjs +89 -5
- package/lib/cli/assistant/tool-registry.mjs +14 -1
- package/lib/cli/assistant/view-model.mjs +132 -0
- package/lib/database/template-steps.mjs +3 -3
- package/lib/runner/runtime-preparation.mjs +36 -0
- package/lib/runner/template.mjs +24 -3
- package/lib/runtime-src/k6/http-checks.js +17 -59
- package/lib/runtime-src/shared/http-check-plan.mjs +53 -0
- package/node_modules/@elench/next-analysis/package.json +1 -1
- package/node_modules/@elench/testkit-bridge/package.json +2 -2
- package/node_modules/@elench/testkit-protocol/package.json +1 -1
- package/node_modules/@elench/ts-analysis/package.json +1 -1
- package/package.json +7 -6
package/README.md
CHANGED
|
@@ -78,11 +78,23 @@ npx @elench/testkit db snapshot capture --service api --output scripts/testkit/s
|
|
|
78
78
|
```
|
|
79
79
|
|
|
80
80
|
`testkit` is assistant-first in an interactive TTY. The interactive assistant
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
81
|
+
opens with a repo-aware landing panel: provider/model, current directory,
|
|
82
|
+
latest run result, focused file/service, regression counts, and suggested next
|
|
83
|
+
prompts. The bottom composer is the primary interaction surface, and the status
|
|
84
|
+
line shows approximate context remaining when the active provider/model window
|
|
85
|
+
is known, for example `[~96% remaining]`.
|
|
86
|
+
|
|
87
|
+
Natural-language turns still go through Codex or Claude, but `testkit` owns the
|
|
88
|
+
transcript, command execution surface, context files under `.testkit/assistant/`,
|
|
89
|
+
and rendering around `testkit`, `npm`, and `npx` commands. When the provider
|
|
90
|
+
runs testkit-managed commands, the assistant records structured tool lifecycle
|
|
91
|
+
blocks and refreshes the latest run artifact so follow-up questions can use the
|
|
92
|
+
new result immediately.
|
|
93
|
+
|
|
94
|
+
Assistant provider coverage is tested against the real `codex` and `claude`
|
|
95
|
+
CLIs. The test suite assumes both are installed and authenticated; provider
|
|
96
|
+
adapter, assistant shell, `shell_exec`, and real testkit-run coverage do not use
|
|
97
|
+
provider stand-in binaries or simulated provider sessions.
|
|
86
98
|
|
|
87
99
|
Assistant runtime settings are repo-local. Use `/provider`, `/model`,
|
|
88
100
|
`/effort`, and `/settings` inside the assistant to inspect or change the active
|
|
@@ -90,7 +102,8 @@ provider runtime; changes are persisted to `.testkit/assistant/settings.json`.
|
|
|
90
102
|
CLI flags such as `--provider`, `--model`, `--effort`, and repeatable
|
|
91
103
|
`--provider-arg` override those settings for the current launch. The composer
|
|
92
104
|
has an always-visible cursor and supports arrow keys, Home/End, Ctrl+A/Ctrl+E,
|
|
93
|
-
Backspace, Delete, and Ctrl+
|
|
105
|
+
Backspace, Delete, Ctrl+D, and Ctrl+L to clear the visible transcript. Ctrl+C
|
|
106
|
+
quits the assistant.
|
|
94
107
|
|
|
95
108
|
The non-interactive `assistant --message ...` mode uses the same provider/tool
|
|
96
109
|
engine for one hosted turn at a time. It is useful in scripts and tests, but
|
|
@@ -268,7 +281,7 @@ File-local execution metadata now lives next to the test when possible:
|
|
|
268
281
|
import { defineFile } from "@elench/testkit/config";
|
|
269
282
|
|
|
270
283
|
export const testkit = defineFile({
|
|
271
|
-
skip: "Billing is
|
|
284
|
+
skip: "Billing is currently unavailable locally",
|
|
272
285
|
locks: ["global-worker-loop"],
|
|
273
286
|
});
|
|
274
287
|
```
|
|
@@ -4,7 +4,6 @@ import {
|
|
|
4
4
|
buildStatusEvent,
|
|
5
5
|
buildToolEvent,
|
|
6
6
|
createHostedSessionRunner,
|
|
7
|
-
extractTextFragments,
|
|
8
7
|
} from "./shared.mjs";
|
|
9
8
|
|
|
10
9
|
export function startClaudeHostedSession({
|
|
@@ -21,6 +20,7 @@ export function startClaudeHostedSession({
|
|
|
21
20
|
"-p",
|
|
22
21
|
"--output-format",
|
|
23
22
|
"stream-json",
|
|
23
|
+
"--verbose",
|
|
24
24
|
"--include-partial-messages",
|
|
25
25
|
];
|
|
26
26
|
|
|
@@ -73,6 +73,25 @@ function parseClaudePayload(payload) {
|
|
|
73
73
|
return events;
|
|
74
74
|
}
|
|
75
75
|
|
|
76
|
+
if (type === "stream_event") {
|
|
77
|
+
const streamEvent = payload.event || {};
|
|
78
|
+
if (streamEvent.type === "content_block_delta" && streamEvent.delta?.type === "text_delta") {
|
|
79
|
+
const text = String(streamEvent.delta.text || "");
|
|
80
|
+
if (text) events.push({ type: "delta", text });
|
|
81
|
+
return events;
|
|
82
|
+
}
|
|
83
|
+
if (streamEvent.type === "tool_use" || streamEvent.content_block?.type === "tool_use") {
|
|
84
|
+
const tool = streamEvent.content_block || streamEvent;
|
|
85
|
+
const event = buildToolEvent(
|
|
86
|
+
tool.name || tool.tool_name || streamEvent.type,
|
|
87
|
+
tool.input ? JSON.stringify(tool.input) : null
|
|
88
|
+
);
|
|
89
|
+
if (event) events.push(event);
|
|
90
|
+
return events;
|
|
91
|
+
}
|
|
92
|
+
return events;
|
|
93
|
+
}
|
|
94
|
+
|
|
76
95
|
if (type && /tool/i.test(type)) {
|
|
77
96
|
const event = buildToolEvent(
|
|
78
97
|
payload.name || payload.tool_name || payload.tool || type,
|
|
@@ -82,10 +101,14 @@ function parseClaudePayload(payload) {
|
|
|
82
101
|
return events;
|
|
83
102
|
}
|
|
84
103
|
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
104
|
+
if (type === "assistant") {
|
|
105
|
+
return events;
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
if (type === "result") {
|
|
109
|
+
if (payload.is_error || payload.subtype === "error") {
|
|
110
|
+
const event = buildErrorEvent(payload.result || payload.error || "Claude command failed");
|
|
111
|
+
if (event) events.push(event);
|
|
89
112
|
}
|
|
90
113
|
return events;
|
|
91
114
|
}
|
|
@@ -22,17 +22,14 @@ export function startCodexHostedSession({
|
|
|
22
22
|
} = {}) {
|
|
23
23
|
const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "testkit-codex-"));
|
|
24
24
|
const outputFile = path.join(tempDir, "final-message.txt");
|
|
25
|
-
const args =
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
}
|
|
33
|
-
args.push(...normalizeProviderArgs(providerArgs));
|
|
34
|
-
|
|
35
|
-
args.push(prompt);
|
|
25
|
+
const args = buildCodexArgs({
|
|
26
|
+
outputFile,
|
|
27
|
+
purpose,
|
|
28
|
+
model,
|
|
29
|
+
providerArgs,
|
|
30
|
+
prompt,
|
|
31
|
+
sandbox: process.env.TESTKIT_CODEX_SANDBOX,
|
|
32
|
+
});
|
|
36
33
|
|
|
37
34
|
const child = execa(command, args, {
|
|
38
35
|
cwd,
|
|
@@ -65,6 +62,28 @@ export function startCodexHostedSession({
|
|
|
65
62
|
};
|
|
66
63
|
}
|
|
67
64
|
|
|
65
|
+
export function buildCodexArgs({
|
|
66
|
+
outputFile,
|
|
67
|
+
purpose = "assistant",
|
|
68
|
+
model = null,
|
|
69
|
+
providerArgs = [],
|
|
70
|
+
prompt = "",
|
|
71
|
+
sandbox = null,
|
|
72
|
+
} = {}) {
|
|
73
|
+
const args = ["exec", "--json"];
|
|
74
|
+
if (outputFile) args.push("-o", outputFile);
|
|
75
|
+
|
|
76
|
+
if (purpose === "assistant") {
|
|
77
|
+
args.push("-s", String(sandbox || "workspace-write"));
|
|
78
|
+
}
|
|
79
|
+
if (model) {
|
|
80
|
+
args.push("--model", String(model));
|
|
81
|
+
}
|
|
82
|
+
args.push(...normalizeProviderArgs(providerArgs));
|
|
83
|
+
args.push(prompt);
|
|
84
|
+
return args;
|
|
85
|
+
}
|
|
86
|
+
|
|
68
87
|
function normalizeProviderArgs(providerArgs) {
|
|
69
88
|
if (!Array.isArray(providerArgs)) return [];
|
|
70
89
|
return providerArgs.flatMap((arg) => String(arg || "").split(/\s+/).filter(Boolean));
|
|
@@ -36,6 +36,11 @@ export function createHostedSessionRunner({ provider, child, onEvent, parsePaylo
|
|
|
36
36
|
const completion = (async () => {
|
|
37
37
|
const result = await child;
|
|
38
38
|
const finalText = (readFinalText ? readFinalText(result) : null) || assistantText.trim() || null;
|
|
39
|
+
if ((result.exitCode ?? 0) !== 0 && !finalText) {
|
|
40
|
+
const message = result.stderr || result.stdout || `${provider} exited with code ${result.exitCode ?? 1}`;
|
|
41
|
+
emit({ type: "error", message });
|
|
42
|
+
throw new Error(message);
|
|
43
|
+
}
|
|
39
44
|
if (finalText) emit({ type: "final", text: finalText });
|
|
40
45
|
emit({ type: "exit", code: result.exitCode ?? 0 });
|
|
41
46
|
settled = true;
|
|
@@ -1,9 +1,10 @@
|
|
|
1
1
|
import React, { createElement, useEffect, useMemo, useState } from "react";
|
|
2
|
-
import { Box, Text, useApp, useInput } from "ink";
|
|
3
|
-
import { bold, dim, green, red, yellow } from "../presentation/colors.mjs";
|
|
2
|
+
import { Box, Text, useApp, useInput, useStdout } from "ink";
|
|
3
|
+
import { bold, cyan, dim, green, red, yellow } from "../presentation/colors.mjs";
|
|
4
4
|
import { getComposerRenderParts } from "./composer.mjs";
|
|
5
|
+
import { buildAssistantViewModel } from "./view-model.mjs";
|
|
5
6
|
|
|
6
|
-
const
|
|
7
|
+
const MAX_BLOCK_LINES = 18;
|
|
7
8
|
|
|
8
9
|
export function AssistantApp({
|
|
9
10
|
assistantState,
|
|
@@ -13,6 +14,7 @@ export function AssistantApp({
|
|
|
13
14
|
onRequestClose,
|
|
14
15
|
} = {}) {
|
|
15
16
|
const { exit } = useApp();
|
|
17
|
+
const { stdout } = useStdout();
|
|
16
18
|
const [snapshot, setSnapshot] = useState(() => assistantState.getSnapshot());
|
|
17
19
|
const [initialPromptStarted, setInitialPromptStarted] = useState(false);
|
|
18
20
|
const [initialPromptFinished, setInitialPromptFinished] = useState(false);
|
|
@@ -30,7 +32,7 @@ export function AssistantApp({
|
|
|
30
32
|
Promise.resolve(assistantState.submitInput(initialPrompt)).finally(() => {
|
|
31
33
|
setInitialPromptFinished(true);
|
|
32
34
|
});
|
|
33
|
-
}, [assistantState,
|
|
35
|
+
}, [assistantState, initialPrompt, initialPromptStarted]);
|
|
34
36
|
|
|
35
37
|
useEffect(() => {
|
|
36
38
|
if (!exitAfterInitialPrompt || !initialPromptFinished || snapshot.busy) return;
|
|
@@ -40,9 +42,12 @@ export function AssistantApp({
|
|
|
40
42
|
return () => clearTimeout(timer);
|
|
41
43
|
}, [exit, exitAfterInitialPrompt, initialPromptFinished, onRequestClose, snapshot.busy]);
|
|
42
44
|
|
|
43
|
-
const
|
|
44
|
-
() => snapshot
|
|
45
|
-
|
|
45
|
+
const view = useMemo(
|
|
46
|
+
() => buildAssistantViewModel(snapshot, {
|
|
47
|
+
cwd: snapshot.productDir || process.cwd(),
|
|
48
|
+
terminalWidth: stdout?.columns || process.stdout?.columns || 100,
|
|
49
|
+
}),
|
|
50
|
+
[snapshot, stdout?.columns]
|
|
46
51
|
);
|
|
47
52
|
|
|
48
53
|
return createElement(
|
|
@@ -55,28 +60,15 @@ export function AssistantApp({
|
|
|
55
60
|
onRequestClose,
|
|
56
61
|
})
|
|
57
62
|
: null,
|
|
58
|
-
|
|
59
|
-
|
|
63
|
+
view.blocks.length === 0
|
|
64
|
+
? createElement(WelcomePanel, { view })
|
|
65
|
+
: createElement(Transcript, { view }),
|
|
60
66
|
createElement(Text, null, ""),
|
|
61
|
-
createElement(
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
createElement(Text, null, ""),
|
|
67
|
-
createElement(
|
|
68
|
-
Box,
|
|
69
|
-
{
|
|
70
|
-
borderStyle: "round",
|
|
71
|
-
flexDirection: "column",
|
|
72
|
-
paddingLeft: 1,
|
|
73
|
-
paddingRight: 1,
|
|
74
|
-
},
|
|
75
|
-
createElement(Text, null, dim("Message")),
|
|
76
|
-
renderComposer(snapshot)
|
|
77
|
-
),
|
|
78
|
-
createElement(Text, null, ""),
|
|
79
|
-
createElement(Text, null, dim(buildFooter(snapshot, initialPromptFinished)))
|
|
67
|
+
createElement(ComposerBar, { view, busy: snapshot.busy }),
|
|
68
|
+
createElement(Text, null, dim(view.statusLine)),
|
|
69
|
+
exitAfterInitialPrompt && initialPromptFinished && !snapshot.busy
|
|
70
|
+
? createElement(Text, null, dim("initial prompt complete"))
|
|
71
|
+
: null
|
|
80
72
|
);
|
|
81
73
|
}
|
|
82
74
|
|
|
@@ -88,8 +80,8 @@ function AssistantInputHandler({ assistantState, snapshot, onRequestClose }) {
|
|
|
88
80
|
(onRequestClose || exit)();
|
|
89
81
|
return;
|
|
90
82
|
}
|
|
91
|
-
if (input === "
|
|
92
|
-
(
|
|
83
|
+
if (key.ctrl && input === "l" && !snapshot.busy) {
|
|
84
|
+
assistantState.clearMessages();
|
|
93
85
|
return;
|
|
94
86
|
}
|
|
95
87
|
if (key.return) {
|
|
@@ -130,76 +122,113 @@ function AssistantInputHandler({ assistantState, snapshot, onRequestClose }) {
|
|
|
130
122
|
return null;
|
|
131
123
|
}
|
|
132
124
|
|
|
133
|
-
function
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
125
|
+
function WelcomePanel({ view }) {
|
|
126
|
+
return createElement(
|
|
127
|
+
Box,
|
|
128
|
+
{
|
|
129
|
+
borderStyle: "round",
|
|
130
|
+
flexDirection: "column",
|
|
131
|
+
paddingLeft: 1,
|
|
132
|
+
paddingRight: 1,
|
|
133
|
+
},
|
|
134
|
+
createElement(Text, null, bold(view.title)),
|
|
135
|
+
createElement(Text, null, dim(view.welcome.subtitle)),
|
|
136
|
+
createElement(Text, null, ""),
|
|
137
|
+
...view.welcome.rows.map(([label, value]) => (
|
|
138
|
+
createElement(Text, { key: label }, `${padLabel(label)} ${colorWelcomeValue(label, value)}`)
|
|
139
|
+
)),
|
|
140
|
+
createElement(Text, null, ""),
|
|
141
|
+
createElement(Text, null, bold("Try")),
|
|
142
|
+
...view.welcome.suggestions.map((suggestion) => (
|
|
143
|
+
createElement(Text, { key: suggestion }, ` ${dim("›")} ${suggestion}`)
|
|
144
|
+
))
|
|
145
|
+
);
|
|
146
|
+
}
|
|
142
147
|
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
)
|
|
148
|
+
function Transcript({ view }) {
|
|
149
|
+
return createElement(
|
|
150
|
+
Box,
|
|
151
|
+
{ flexDirection: "column" },
|
|
152
|
+
createElement(Text, null, bold(view.title)),
|
|
153
|
+
createElement(Text, null, dim(view.welcome.rows.find(([label]) => label === "Provider")?.[1] || "")),
|
|
154
|
+
createElement(Text, null, ""),
|
|
155
|
+
view.notice ? createElement(Text, null, yellow(view.notice)) : null,
|
|
156
|
+
...view.blocks.flatMap((block) => renderBlock(block))
|
|
157
|
+
);
|
|
158
|
+
}
|
|
159
|
+
|
|
160
|
+
function renderBlock(block) {
|
|
161
|
+
const lines = String(block.text || "").split(/\r?\n/);
|
|
162
|
+
const visibleLines = lines.length > MAX_BLOCK_LINES
|
|
163
|
+
? [...lines.slice(0, MAX_BLOCK_LINES - 1), `… ${lines.length - MAX_BLOCK_LINES + 1} more lines omitted`]
|
|
164
|
+
: lines;
|
|
165
|
+
const marker = colorMarker(block);
|
|
166
|
+
const title = block.title ? ` ${bold(block.title)}` : "";
|
|
167
|
+
const first = visibleLines[0] || "";
|
|
168
|
+
const rendered = [
|
|
169
|
+
createElement(Text, { key: `${block.id}-first` }, `${marker}${title}${title && first ? " " : ""}${colorBlockText(block, first)}`),
|
|
170
|
+
];
|
|
171
|
+
|
|
172
|
+
for (let index = 1; index < visibleLines.length; index += 1) {
|
|
173
|
+
rendered.push(createElement(Text, { key: `${block.id}-${index}` }, ` ${visibleLines[index]}`));
|
|
152
174
|
}
|
|
153
|
-
rendered.push(createElement(Text, { key: `${
|
|
175
|
+
rendered.push(createElement(Text, { key: `${block.id}-gap` }, ""));
|
|
154
176
|
return rendered;
|
|
155
177
|
}
|
|
156
178
|
|
|
157
|
-
function
|
|
179
|
+
function ComposerBar({ view, busy }) {
|
|
158
180
|
const { before, current, after, empty } = getComposerRenderParts({
|
|
159
|
-
text:
|
|
160
|
-
cursor:
|
|
181
|
+
text: view.composer.text,
|
|
182
|
+
cursor: view.composer.cursor,
|
|
161
183
|
});
|
|
184
|
+
const prompt = cyan("❯");
|
|
185
|
+
const promptText = empty ? dim(`${view.composer.placeholder} `) : before;
|
|
162
186
|
return createElement(
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
187
|
+
Box,
|
|
188
|
+
{
|
|
189
|
+
borderStyle: "single",
|
|
190
|
+
borderLeft: false,
|
|
191
|
+
borderRight: false,
|
|
192
|
+
paddingTop: 0,
|
|
193
|
+
paddingBottom: 0,
|
|
194
|
+
},
|
|
195
|
+
createElement(
|
|
196
|
+
Text,
|
|
197
|
+
null,
|
|
198
|
+
`${prompt} `,
|
|
199
|
+
promptText,
|
|
200
|
+
createElement(Text, { inverse: true }, current),
|
|
201
|
+
after,
|
|
202
|
+
busy ? dim(" provider responding") : ""
|
|
203
|
+
)
|
|
168
204
|
);
|
|
169
205
|
}
|
|
170
206
|
|
|
171
|
-
function
|
|
172
|
-
|
|
173
|
-
const provider = snapshot.provider || "auto";
|
|
174
|
-
const resolvedProvider = snapshot.resolvedProvider && snapshot.resolvedProvider !== provider ? `→${snapshot.resolvedProvider}` : "";
|
|
175
|
-
const model = snapshot.model ? ` · ${snapshot.model}` : "";
|
|
176
|
-
const effort = snapshot.effort ? ` · ${snapshot.effort}` : "";
|
|
177
|
-
const context = snapshot.context?.selection?.filePath || snapshot.context?.selection?.serviceName || "no focus";
|
|
178
|
-
return `testkit assistant · ${provider}${resolvedProvider}${model}${effort} · ${status} · ${context}`;
|
|
207
|
+
function padLabel(label) {
|
|
208
|
+
return `${dim(String(label).padEnd(10, " "))}`;
|
|
179
209
|
}
|
|
180
210
|
|
|
181
|
-
function
|
|
182
|
-
if (
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
if (
|
|
186
|
-
|
|
187
|
-
}
|
|
188
|
-
return "Enter send · arrows/Home/End move cursor · Backspace/Delete edit · /settings · q quit";
|
|
211
|
+
function colorWelcomeValue(label, value) {
|
|
212
|
+
if (label === "Latest" && /^FAILED\b/.test(String(value))) return red(value);
|
|
213
|
+
if (label === "Latest" && /^PASSED\b/.test(String(value))) return green(value);
|
|
214
|
+
if (label === "Issues" && value !== "None detected") return yellow(value);
|
|
215
|
+
if (label === "Provider") return cyan(value);
|
|
216
|
+
return value;
|
|
189
217
|
}
|
|
190
218
|
|
|
191
|
-
function
|
|
192
|
-
if (
|
|
193
|
-
if (
|
|
194
|
-
if (
|
|
195
|
-
|
|
219
|
+
function colorMarker(block) {
|
|
220
|
+
if (block.kind === "user") return cyan(block.marker);
|
|
221
|
+
if (block.kind === "system") return red(block.marker);
|
|
222
|
+
if (block.kind === "tool-running") return yellow(block.marker);
|
|
223
|
+
if (block.kind === "testkit-run") return green(block.marker);
|
|
224
|
+
return block.marker;
|
|
196
225
|
}
|
|
197
226
|
|
|
198
|
-
function
|
|
199
|
-
if (
|
|
200
|
-
if (
|
|
201
|
-
if (
|
|
202
|
-
return
|
|
227
|
+
function colorBlockText(block, text) {
|
|
228
|
+
if (block.kind === "user") return text;
|
|
229
|
+
if (block.kind === "system") return red(text);
|
|
230
|
+
if (block.kind === "tool-running") return yellow(text);
|
|
231
|
+
return text;
|
|
203
232
|
}
|
|
204
233
|
|
|
205
234
|
function isPrintableInput(input, key) {
|
|
@@ -160,7 +160,8 @@ function buildContextMarkdown(productDir, snapshot, paths) {
|
|
|
160
160
|
lines.push(
|
|
161
161
|
"",
|
|
162
162
|
"## Guidance",
|
|
163
|
-
"- Use shell commands like `npm run testkit`, `npx testkit`, or `testkit run --dir
|
|
163
|
+
"- Use shell commands like `npm run testkit`, `npx testkit`, or `testkit run <type> --dir .` when you need to execute tests.",
|
|
164
|
+
"- Do not reinterpret CLI syntax after an execution failure unless `testkit run --help` confirms a syntax problem.",
|
|
164
165
|
"- Use the command log and focused context files before rereading artifacts manually.",
|
|
165
166
|
"- Prefer repo-local commands over guessing project-specific wrappers.",
|
|
166
167
|
""
|
|
@@ -173,15 +174,15 @@ function buildCommandsMarkdown() {
|
|
|
173
174
|
return [
|
|
174
175
|
"# Testkit Commands",
|
|
175
176
|
"",
|
|
176
|
-
"- `testkit run --dir
|
|
177
|
-
"- `testkit run --dir
|
|
177
|
+
"- `testkit run int --dir .`",
|
|
178
|
+
"- `testkit run e2e --dir .`",
|
|
178
179
|
"- `testkit run --dir . --file path/to/file.testkit.ts`",
|
|
179
180
|
"- `testkit discover --dir .`",
|
|
180
181
|
"- `testkit status --dir .`",
|
|
181
182
|
"- `testkit doctor --dir .`",
|
|
182
183
|
"- `testkit destroy --dir .`",
|
|
183
184
|
"- `npm run testkit`",
|
|
184
|
-
"- `npx testkit
|
|
185
|
+
"- `npx testkit run e2e --dir .`",
|
|
185
186
|
"",
|
|
186
187
|
].join("\n");
|
|
187
188
|
}
|
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
const DEFAULT_CHARS_PER_TOKEN = 4;
|
|
2
|
+
|
|
3
|
+
const MODEL_WINDOWS = [
|
|
4
|
+
[/claude.*opus.*4\.7/i, 1_000_000],
|
|
5
|
+
[/claude.*sonnet.*4/i, 200_000],
|
|
6
|
+
[/claude.*haiku.*4/i, 200_000],
|
|
7
|
+
[/claude/i, 200_000],
|
|
8
|
+
[/gpt-5\.5/i, 400_000],
|
|
9
|
+
[/gpt-5\.4/i, 400_000],
|
|
10
|
+
[/gpt-5\.3/i, 400_000],
|
|
11
|
+
[/gpt-5\.2/i, 400_000],
|
|
12
|
+
[/gpt-5\b/i, 400_000],
|
|
13
|
+
[/codex/i, 400_000],
|
|
14
|
+
];
|
|
15
|
+
|
|
16
|
+
export function resolveContextWindow({ provider, model } = {}) {
|
|
17
|
+
const label = [provider, model].filter(Boolean).join(" ");
|
|
18
|
+
for (const [pattern, tokens] of MODEL_WINDOWS) {
|
|
19
|
+
if (pattern.test(label)) return tokens;
|
|
20
|
+
}
|
|
21
|
+
return null;
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
export function estimateTokenCount(text) {
|
|
25
|
+
const value = String(text || "");
|
|
26
|
+
if (!value) return 0;
|
|
27
|
+
return Math.max(1, Math.ceil(value.length / DEFAULT_CHARS_PER_TOKEN));
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
export function buildContextUsage({
|
|
31
|
+
provider,
|
|
32
|
+
model,
|
|
33
|
+
prompt,
|
|
34
|
+
exactUsedTokens = null,
|
|
35
|
+
exactMaxTokens = null,
|
|
36
|
+
} = {}) {
|
|
37
|
+
const maxTokens = normalizePositiveInteger(exactMaxTokens) || resolveContextWindow({ provider, model });
|
|
38
|
+
const usedTokens = normalizePositiveInteger(exactUsedTokens) || estimateTokenCount(prompt);
|
|
39
|
+
if (!maxTokens || !usedTokens) {
|
|
40
|
+
return {
|
|
41
|
+
known: false,
|
|
42
|
+
estimated: true,
|
|
43
|
+
usedTokens: usedTokens || null,
|
|
44
|
+
maxTokens: maxTokens || null,
|
|
45
|
+
remainingPercent: null,
|
|
46
|
+
};
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
const clampedUsed = Math.min(usedTokens, maxTokens);
|
|
50
|
+
return {
|
|
51
|
+
known: true,
|
|
52
|
+
estimated: !exactUsedTokens,
|
|
53
|
+
usedTokens: clampedUsed,
|
|
54
|
+
maxTokens,
|
|
55
|
+
remainingPercent: Math.max(0, Math.floor(((maxTokens - clampedUsed) / maxTokens) * 100)),
|
|
56
|
+
};
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
export function formatContextRemaining(usage) {
|
|
60
|
+
if (!usage?.known || usage.remainingPercent == null) return "[context unknown]";
|
|
61
|
+
const prefix = usage.estimated ? "~" : "";
|
|
62
|
+
return `[${prefix}${usage.remainingPercent}% remaining]`;
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
function normalizePositiveInteger(value) {
|
|
66
|
+
const number = Number(value);
|
|
67
|
+
if (!Number.isFinite(number) || number <= 0) return null;
|
|
68
|
+
return Math.floor(number);
|
|
69
|
+
}
|
|
@@ -16,6 +16,7 @@ export async function runAssistantConversationTurn({
|
|
|
16
16
|
onStatus,
|
|
17
17
|
onToolEvent,
|
|
18
18
|
onResolvedProvider,
|
|
19
|
+
onPrompt,
|
|
19
20
|
} = {}) {
|
|
20
21
|
const tools = listAssistantTools();
|
|
21
22
|
const toolContext = {
|
|
@@ -43,6 +44,12 @@ export async function runAssistantConversationTurn({
|
|
|
43
44
|
const runtimeSettings = settings || { provider };
|
|
44
45
|
const resolvedProvider = resolvePreferredProvider(runtimeSettings.provider || provider, env);
|
|
45
46
|
onResolvedProvider?.(resolvedProvider);
|
|
47
|
+
onPrompt?.({
|
|
48
|
+
prompt,
|
|
49
|
+
provider: resolvedProvider,
|
|
50
|
+
model: runtimeSettings.model || null,
|
|
51
|
+
effort: runtimeSettings.effort || null,
|
|
52
|
+
});
|
|
46
53
|
onStatus?.(`Thinking with ${resolvedProvider}...`);
|
|
47
54
|
const events = [];
|
|
48
55
|
const session = startAgentSession({
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import { loadCurrentRunArtifact, loadLatestRunArtifact } from "../viewer.mjs";
|
|
2
2
|
import { createInspectState } from "../tui/inspect-state.mjs";
|
|
3
3
|
import { buildContextSelection } from "../context-resources.mjs";
|
|
4
|
+
import { isProviderInstalled } from "../agents/index.mjs";
|
|
4
5
|
import { parseSlashCommand, formatSlashHelpLines } from "./slash-commands.mjs";
|
|
5
6
|
import { executeAssistantTool } from "./tool-registry.mjs";
|
|
6
7
|
import { runAssistantConversationTurn } from "./session.mjs";
|
|
@@ -22,6 +23,7 @@ import {
|
|
|
22
23
|
moveComposerCursorToStart as moveComposerCursorStateToStart,
|
|
23
24
|
setComposerText,
|
|
24
25
|
} from "./composer.mjs";
|
|
26
|
+
import { buildContextUsage } from "./context-window.mjs";
|
|
25
27
|
|
|
26
28
|
export function createAssistantState({
|
|
27
29
|
productDir,
|
|
@@ -54,8 +56,13 @@ export function createAssistantState({
|
|
|
54
56
|
providerArgs,
|
|
55
57
|
}
|
|
56
58
|
);
|
|
57
|
-
let resolvedProviderName =
|
|
59
|
+
let resolvedProviderName = resolveInitialProvider(settings.provider, env);
|
|
58
60
|
let activeStatus = null;
|
|
61
|
+
let contextUsage = buildContextUsage({
|
|
62
|
+
provider: resolvedProviderName || settings.provider,
|
|
63
|
+
model: settings.model,
|
|
64
|
+
prompt: "",
|
|
65
|
+
});
|
|
59
66
|
|
|
60
67
|
inspectState.subscribe(() => {
|
|
61
68
|
commandLog.refresh();
|
|
@@ -170,11 +177,19 @@ export function createAssistantState({
|
|
|
170
177
|
setProvider(nextProvider) {
|
|
171
178
|
settings = mergeAssistantSettings(settings, { provider: nextProvider || DEFAULT_ASSISTANT_SETTINGS.provider });
|
|
172
179
|
resolvedProviderName = null;
|
|
180
|
+
if (settings.model && getModelProviderMismatch(resolveInitialProvider(settings.provider, env), settings.model)) {
|
|
181
|
+
settings = mergeAssistantSettings(settings, { model: null });
|
|
182
|
+
}
|
|
173
183
|
saveAssistantSettings(productDir, settings);
|
|
174
184
|
notify();
|
|
175
185
|
},
|
|
176
186
|
|
|
177
187
|
setModel(nextModel) {
|
|
188
|
+
const resolvedProvider = resolveInitialProvider(settings.provider, env);
|
|
189
|
+
const mismatch = getModelProviderMismatch(resolvedProvider, nextModel);
|
|
190
|
+
if (mismatch) {
|
|
191
|
+
throw new Error(mismatch);
|
|
192
|
+
}
|
|
178
193
|
settings = mergeAssistantSettings(settings, { model: nextModel || null });
|
|
179
194
|
saveAssistantSettings(productDir, settings);
|
|
180
195
|
notify();
|
|
@@ -270,6 +285,17 @@ export function createAssistantState({
|
|
|
270
285
|
resolvedProviderName = provider;
|
|
271
286
|
notify();
|
|
272
287
|
},
|
|
288
|
+
onPrompt(meta) {
|
|
289
|
+
contextUsage = buildContextUsage({
|
|
290
|
+
provider: meta.provider || settings.provider,
|
|
291
|
+
model: meta.model || settings.model,
|
|
292
|
+
prompt: meta.prompt,
|
|
293
|
+
});
|
|
294
|
+
notify();
|
|
295
|
+
},
|
|
296
|
+
onToolEvent(event) {
|
|
297
|
+
handleAssistantToolEvent(event, appendMessage);
|
|
298
|
+
},
|
|
273
299
|
});
|
|
274
300
|
for (const message of emitted) appendMessage(message);
|
|
275
301
|
} catch (error) {
|
|
@@ -291,6 +317,8 @@ export function createAssistantState({
|
|
|
291
317
|
getSnapshot() {
|
|
292
318
|
return {
|
|
293
319
|
context: buildContextSelection(inspectState.getSnapshot()),
|
|
320
|
+
inspect: inspectState.getSnapshot(),
|
|
321
|
+
productDir,
|
|
294
322
|
messages: [...messages],
|
|
295
323
|
composer: composerState.text,
|
|
296
324
|
composerCursor: composerState.cursor,
|
|
@@ -302,6 +330,7 @@ export function createAssistantState({
|
|
|
302
330
|
effort: settings.effort,
|
|
303
331
|
providerArgs: [...settings.providerArgs],
|
|
304
332
|
activeStatus,
|
|
333
|
+
contextUsage,
|
|
305
334
|
contextPaths: {
|
|
306
335
|
contextPath: commandLog.contextPath,
|
|
307
336
|
summaryPath: commandLog.summaryPath,
|
|
@@ -317,6 +346,29 @@ export function createAssistantState({
|
|
|
317
346
|
return state;
|
|
318
347
|
}
|
|
319
348
|
|
|
349
|
+
function resolveInitialProvider(provider, env) {
|
|
350
|
+
if (provider && provider !== "auto") return provider;
|
|
351
|
+
if (isProviderInstalled("codex", env)) return "codex";
|
|
352
|
+
if (isProviderInstalled("claude", env)) return "claude";
|
|
353
|
+
return null;
|
|
354
|
+
}
|
|
355
|
+
|
|
356
|
+
function getModelProviderMismatch(provider, model) {
|
|
357
|
+
const normalizedModel = String(model || "").trim().toLowerCase();
|
|
358
|
+
if (!provider || !normalizedModel) return null;
|
|
359
|
+
|
|
360
|
+
const looksClaude = /\b(?:opus|sonnet|haiku|claude)\b/.test(normalizedModel);
|
|
361
|
+
const looksCodex = /\b(?:gpt|codex|o[1-9]|chatgpt)\b/.test(normalizedModel);
|
|
362
|
+
|
|
363
|
+
if (provider === "codex" && looksClaude) {
|
|
364
|
+
return `Model "${model}" looks like a Claude model, but the assistant is using Codex. Run /provider claude or /model default.`;
|
|
365
|
+
}
|
|
366
|
+
if (provider === "claude" && looksCodex) {
|
|
367
|
+
return `Model "${model}" looks like a Codex/OpenAI model, but the assistant is using Claude. Run /provider codex or /model default.`;
|
|
368
|
+
}
|
|
369
|
+
return null;
|
|
370
|
+
}
|
|
371
|
+
|
|
320
372
|
async function executeSlashCommand({
|
|
321
373
|
slash,
|
|
322
374
|
state,
|
|
@@ -385,7 +437,18 @@ async function executeSlashCommand({
|
|
|
385
437
|
env,
|
|
386
438
|
commandLog: state.commandLog,
|
|
387
439
|
onEvent(event) {
|
|
388
|
-
if (event.type === "tool-
|
|
440
|
+
if (event.type === "tool-start") {
|
|
441
|
+
appendMessage({
|
|
442
|
+
role: "tool",
|
|
443
|
+
status: "running",
|
|
444
|
+
title: event.title || event.tool || "Tool",
|
|
445
|
+
text: event.message,
|
|
446
|
+
data: {
|
|
447
|
+
command: event.command || null,
|
|
448
|
+
testkitRelated: Boolean(event.testkitRelated),
|
|
449
|
+
},
|
|
450
|
+
});
|
|
451
|
+
} else if (event.type === "tool-status") {
|
|
389
452
|
state.setNotice(event.message);
|
|
390
453
|
}
|
|
391
454
|
},
|
|
@@ -400,6 +463,20 @@ async function executeSlashCommand({
|
|
|
400
463
|
});
|
|
401
464
|
}
|
|
402
465
|
|
|
466
|
+
function handleAssistantToolEvent(event, appendMessage) {
|
|
467
|
+
if (!event || event.type !== "tool-start") return;
|
|
468
|
+
appendMessage({
|
|
469
|
+
role: "tool",
|
|
470
|
+
status: "running",
|
|
471
|
+
title: event.title || event.tool || "Tool",
|
|
472
|
+
text: event.message || "Running tool",
|
|
473
|
+
data: {
|
|
474
|
+
command: event.command || null,
|
|
475
|
+
testkitRelated: Boolean(event.testkitRelated),
|
|
476
|
+
},
|
|
477
|
+
});
|
|
478
|
+
}
|
|
479
|
+
|
|
403
480
|
function formatSettings(snapshot) {
|
|
404
481
|
const rows = [
|
|
405
482
|
["Provider", snapshot.provider || "auto"],
|
|
@@ -443,9 +520,16 @@ async function executeSlashTool(slash, context) {
|
|
|
443
520
|
}
|
|
444
521
|
|
|
445
522
|
function buildRunSlashCommand(options = {}) {
|
|
446
|
-
const
|
|
447
|
-
|
|
448
|
-
|
|
523
|
+
const types = options.type || [];
|
|
524
|
+
const parts = ["testkit", "run"];
|
|
525
|
+
if (types.length === 1) {
|
|
526
|
+
parts.push(types[0]);
|
|
527
|
+
}
|
|
528
|
+
parts.push("--dir", ".");
|
|
529
|
+
if (types.length !== 1) {
|
|
530
|
+
for (const type of types) {
|
|
531
|
+
parts.push("--type", type);
|
|
532
|
+
}
|
|
449
533
|
}
|
|
450
534
|
for (const suite of options.suite || []) {
|
|
451
535
|
parts.push("--suite", suite);
|
|
@@ -61,8 +61,11 @@ async function shellExecTool(args, context) {
|
|
|
61
61
|
raw: command,
|
|
62
62
|
});
|
|
63
63
|
context.onEvent?.({
|
|
64
|
-
type: "tool-
|
|
64
|
+
type: "tool-start",
|
|
65
65
|
tool: "shell_exec",
|
|
66
|
+
command,
|
|
67
|
+
title: shellCommand.title,
|
|
68
|
+
testkitRelated: shellCommand.testkitRelated,
|
|
66
69
|
message: `Running ${shellCommand.display}`,
|
|
67
70
|
});
|
|
68
71
|
|
|
@@ -87,6 +90,16 @@ async function shellExecTool(args, context) {
|
|
|
87
90
|
code: result.exitCode ?? 0,
|
|
88
91
|
signal: result.signal ?? null,
|
|
89
92
|
});
|
|
93
|
+
context.onEvent?.({
|
|
94
|
+
type: "tool-exit",
|
|
95
|
+
tool: "shell_exec",
|
|
96
|
+
command,
|
|
97
|
+
title: shellCommand.title,
|
|
98
|
+
testkitRelated: shellCommand.testkitRelated,
|
|
99
|
+
code: result.exitCode ?? 0,
|
|
100
|
+
signal: result.signal ?? null,
|
|
101
|
+
message: `${shellCommand.display} exited ${result.exitCode ?? 0}`,
|
|
102
|
+
});
|
|
90
103
|
|
|
91
104
|
if (shellCommand.testkitRelated) {
|
|
92
105
|
refreshArtifactSelection(context);
|
|
@@ -0,0 +1,132 @@
|
|
|
1
|
+
import path from "path";
|
|
2
|
+
import { formatContextRemaining } from "./context-window.mjs";
|
|
3
|
+
|
|
4
|
+
const MAX_TRANSCRIPT_BLOCKS = 18;
|
|
5
|
+
|
|
6
|
+
export function buildAssistantViewModel(snapshot, { cwd = process.cwd(), terminalWidth = 100 } = {}) {
|
|
7
|
+
const providerLabel = buildProviderLabel(snapshot);
|
|
8
|
+
const repoName = path.basename(cwd || process.cwd()) || "repository";
|
|
9
|
+
return {
|
|
10
|
+
title: `testkit · ${repoName}`,
|
|
11
|
+
welcome: buildWelcomeModel(snapshot, { cwd, providerLabel }),
|
|
12
|
+
blocks: buildTranscriptBlocks(snapshot.messages || []),
|
|
13
|
+
composer: {
|
|
14
|
+
text: snapshot.composer || "",
|
|
15
|
+
cursor: snapshot.composerCursor ?? 0,
|
|
16
|
+
placeholder: "Ask testkit to run, inspect, or explain something",
|
|
17
|
+
},
|
|
18
|
+
statusLine: buildStatusLine(snapshot, { cwd, providerLabel }),
|
|
19
|
+
busy: Boolean(snapshot.busy),
|
|
20
|
+
notice: snapshot.notice || null,
|
|
21
|
+
terminalWidth,
|
|
22
|
+
};
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
export function buildWelcomeModel(snapshot, { cwd = process.cwd(), providerLabel = null } = {}) {
|
|
26
|
+
const summaryRows = snapshot?.inspect?.summaryData?.rows || snapshot?.summaryData?.rows || [];
|
|
27
|
+
const rowValue = (label) => summaryRows.find(([key]) => key === label)?.[1] || null;
|
|
28
|
+
const contextSelection = snapshot?.context?.selection || {};
|
|
29
|
+
const latestResult = rowValue("Result");
|
|
30
|
+
const counts = [
|
|
31
|
+
rowValue("Passed") ? `${rowValue("Passed")} passed` : null,
|
|
32
|
+
rowValue("Failed") ? `${rowValue("Failed")} failed` : null,
|
|
33
|
+
rowValue("Skipped") ? `${rowValue("Skipped")} skipped` : null,
|
|
34
|
+
].filter(Boolean);
|
|
35
|
+
const issues = [
|
|
36
|
+
rowValue("New regressions") ? `${rowValue("New regressions")} new regression${rowValue("New regressions") === "1" ? "" : "s"}` : null,
|
|
37
|
+
rowValue("Known regressions") ? `${rowValue("Known regressions")} known` : null,
|
|
38
|
+
rowValue("Catalog stale") ? `${rowValue("Catalog stale")} stale` : null,
|
|
39
|
+
].filter(Boolean);
|
|
40
|
+
|
|
41
|
+
return {
|
|
42
|
+
subtitle: "Local testing assistant",
|
|
43
|
+
rows: [
|
|
44
|
+
["Provider", providerLabel || buildProviderLabel(snapshot)],
|
|
45
|
+
["Directory", shortenHome(cwd)],
|
|
46
|
+
["Latest", latestResult ? [latestResult, ...counts].join(" · ") : "No run artifact yet"],
|
|
47
|
+
["Focus", contextSelection.filePath || contextSelection.serviceName || "No focus"],
|
|
48
|
+
["Issues", issues.length ? issues.join(" · ") : "None detected"],
|
|
49
|
+
],
|
|
50
|
+
suggestions: buildSuggestions({ latestResult, contextSelection, hasArtifact: Boolean(latestResult) }),
|
|
51
|
+
};
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
export function buildTranscriptBlocks(messages) {
|
|
55
|
+
return (messages || []).slice(-MAX_TRANSCRIPT_BLOCKS).map((message) => {
|
|
56
|
+
const role = message.role || "system";
|
|
57
|
+
if (role === "tool") {
|
|
58
|
+
return {
|
|
59
|
+
id: message.id,
|
|
60
|
+
kind: classifyToolBlock(message),
|
|
61
|
+
marker: "●",
|
|
62
|
+
title: message.title || message.toolName || "Tool",
|
|
63
|
+
text: message.text || "",
|
|
64
|
+
status: message.status || null,
|
|
65
|
+
command: message.data?.command || null,
|
|
66
|
+
exitCode: message.data?.exitCode ?? null,
|
|
67
|
+
};
|
|
68
|
+
}
|
|
69
|
+
if (role === "user") {
|
|
70
|
+
return {
|
|
71
|
+
id: message.id,
|
|
72
|
+
kind: "user",
|
|
73
|
+
marker: "❯",
|
|
74
|
+
text: message.text || "",
|
|
75
|
+
};
|
|
76
|
+
}
|
|
77
|
+
if (role === "assistant") {
|
|
78
|
+
return {
|
|
79
|
+
id: message.id,
|
|
80
|
+
kind: "assistant",
|
|
81
|
+
marker: "●",
|
|
82
|
+
text: message.text || "",
|
|
83
|
+
};
|
|
84
|
+
}
|
|
85
|
+
return {
|
|
86
|
+
id: message.id,
|
|
87
|
+
kind: "system",
|
|
88
|
+
marker: "!",
|
|
89
|
+
text: message.text || "",
|
|
90
|
+
};
|
|
91
|
+
});
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
export function buildStatusLine(snapshot, { cwd = process.cwd(), providerLabel = null } = {}) {
|
|
95
|
+
const context = formatContextRemaining(snapshot.contextUsage);
|
|
96
|
+
const provider = providerLabel || buildProviderLabel(snapshot);
|
|
97
|
+
const status = snapshot.busy ? snapshot.activeStatus || "working" : "/settings";
|
|
98
|
+
return `${context} ${shortenHome(cwd)} · ${provider} · ${status}`;
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
export function buildProviderLabel(snapshot) {
|
|
102
|
+
const provider = snapshot?.provider || "auto";
|
|
103
|
+
const resolved = snapshot?.resolvedProvider && snapshot.resolvedProvider !== provider ? `→${snapshot.resolvedProvider}` : "";
|
|
104
|
+
const model = snapshot?.model ? ` ${snapshot.model}` : "";
|
|
105
|
+
const effort = snapshot?.effort ? ` ${snapshot.effort}` : "";
|
|
106
|
+
return `${provider}${resolved}${model}${effort}`.trim();
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
function buildSuggestions({ latestResult, contextSelection, hasArtifact }) {
|
|
110
|
+
if (!hasArtifact) {
|
|
111
|
+
return ["Run all tests", "Discover tests", "Run doctor checks"];
|
|
112
|
+
}
|
|
113
|
+
if (latestResult === "FAILED") {
|
|
114
|
+
const suggestions = ["Explain the latest failure", "Show new regressions", "Inspect logs"];
|
|
115
|
+
if (contextSelection?.filePath) suggestions.push(`Inspect ${path.basename(contextSelection.filePath)}`);
|
|
116
|
+
return suggestions;
|
|
117
|
+
}
|
|
118
|
+
return ["Run e2e tests", "Show latest summary", "List test files"];
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
function classifyToolBlock(message) {
|
|
122
|
+
if (message.status === "running") return "tool-running";
|
|
123
|
+
if (message.data?.testkitRelated) return "testkit-run";
|
|
124
|
+
return "tool-result";
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
function shortenHome(value) {
|
|
128
|
+
const text = String(value || "");
|
|
129
|
+
const home = process.env.HOME;
|
|
130
|
+
if (home && text.startsWith(home)) return `~${text.slice(home.length)}`;
|
|
131
|
+
return text;
|
|
132
|
+
}
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import fs from "fs";
|
|
2
2
|
import path from "path";
|
|
3
3
|
import { execa } from "execa";
|
|
4
|
-
import {
|
|
4
|
+
import { buildTemplateExecutionEnv } from "../runner/template.mjs";
|
|
5
5
|
import {
|
|
6
6
|
collectConfiguredInputs,
|
|
7
7
|
runConfiguredSteps,
|
|
@@ -13,7 +13,7 @@ export async function runTemplateStage(config, stageName, databaseUrl, options =
|
|
|
13
13
|
if (steps.length === 0) return;
|
|
14
14
|
|
|
15
15
|
const env = {
|
|
16
|
-
...
|
|
16
|
+
...buildTemplateExecutionEnv(config, {}, process.env),
|
|
17
17
|
DATABASE_URL: databaseUrl,
|
|
18
18
|
};
|
|
19
19
|
|
|
@@ -54,7 +54,7 @@ export async function captureTemplateSnapshot(config, outputPath, databaseUrl, o
|
|
|
54
54
|
{
|
|
55
55
|
cwd: config.productDir,
|
|
56
56
|
env: {
|
|
57
|
-
...
|
|
57
|
+
...buildTemplateExecutionEnv(config, {}, process.env),
|
|
58
58
|
DATABASE_URL: templateDbUrl,
|
|
59
59
|
},
|
|
60
60
|
stdout: "pipe",
|
|
@@ -127,6 +127,7 @@ export async function computeRuntimePrepareFingerprint(config) {
|
|
|
127
127
|
: null,
|
|
128
128
|
})
|
|
129
129
|
);
|
|
130
|
+
hash.update(JSON.stringify(collectRuntimeDatabaseFingerprintInputs(config)));
|
|
130
131
|
|
|
131
132
|
for (const envFile of config.testkit.envFiles || []) {
|
|
132
133
|
appendFileToHash(hash, config.productDir, resolveServiceCwd(config.productDir, envFile));
|
|
@@ -138,6 +139,41 @@ export async function computeRuntimePrepareFingerprint(config) {
|
|
|
138
139
|
return hash.digest("hex");
|
|
139
140
|
}
|
|
140
141
|
|
|
142
|
+
function collectRuntimeDatabaseFingerprintInputs(config) {
|
|
143
|
+
const inputs = [];
|
|
144
|
+
const ownDatabaseUrl = readDatabaseUrl(config.stateDir);
|
|
145
|
+
if (ownDatabaseUrl) {
|
|
146
|
+
inputs.push({ service: config.name, url: ownDatabaseUrl });
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
const referencedServices = new Set();
|
|
150
|
+
for (const value of Object.values(config.testkit.serviceEnv || {})) {
|
|
151
|
+
collectDatabasePlaceholderServices(value, referencedServices, config.name);
|
|
152
|
+
}
|
|
153
|
+
for (const value of Object.values(config.testkit.local?.env || {})) {
|
|
154
|
+
collectDatabasePlaceholderServices(value, referencedServices, config.name);
|
|
155
|
+
}
|
|
156
|
+
|
|
157
|
+
const stateDirByService = config.testkit.templateContext?.stateDirByService;
|
|
158
|
+
for (const serviceName of [...referencedServices].sort()) {
|
|
159
|
+
const stateDir = stateDirByService?.get?.(serviceName);
|
|
160
|
+
const databaseUrl = stateDir ? readDatabaseUrl(stateDir) : null;
|
|
161
|
+
inputs.push({ service: serviceName, url: databaseUrl || null });
|
|
162
|
+
}
|
|
163
|
+
|
|
164
|
+
return inputs;
|
|
165
|
+
}
|
|
166
|
+
|
|
167
|
+
function collectDatabasePlaceholderServices(value, out, defaultServiceName) {
|
|
168
|
+
if (typeof value !== "string") return;
|
|
169
|
+
const matcher = /\{db(?:Url|Host|Port|Name|User|Password)(?::([a-zA-Z0-9_-]+))?\}/g;
|
|
170
|
+
let match = matcher.exec(value);
|
|
171
|
+
while (match) {
|
|
172
|
+
out.add(match[1] || defaultServiceName);
|
|
173
|
+
match = matcher.exec(value);
|
|
174
|
+
}
|
|
175
|
+
}
|
|
176
|
+
|
|
141
177
|
function appendResolvedInputToHash(hash, productDir, absPath) {
|
|
142
178
|
const relative = path.relative(productDir, absPath);
|
|
143
179
|
appendInputToHash(hash, productDir, relative);
|
package/lib/runner/template.mjs
CHANGED
|
@@ -204,17 +204,29 @@ export function buildExecutionEnv(config, extraEnv = {}, processEnv = process.en
|
|
|
204
204
|
return buildExecutionEnvWithContext(config, null, extraEnv, processEnv);
|
|
205
205
|
}
|
|
206
206
|
|
|
207
|
+
export function buildTemplateExecutionEnv(config, extraEnv = {}, processEnv = process.env) {
|
|
208
|
+
return buildExecutionEnvWithContext(config, null, extraEnv, processEnv, {
|
|
209
|
+
omitRuntimeDatabaseBindings: true,
|
|
210
|
+
});
|
|
211
|
+
}
|
|
212
|
+
|
|
207
213
|
export function buildTaskExecutionEnv(config, lease, extraEnv = {}, processEnv = process.env) {
|
|
208
214
|
return buildExecutionEnvWithContext(config, lease, extraEnv, processEnv);
|
|
209
215
|
}
|
|
210
216
|
|
|
211
|
-
function buildExecutionEnvWithContext(config, lease, extraEnv, processEnv) {
|
|
217
|
+
function buildExecutionEnvWithContext(config, lease, extraEnv, processEnv, options = {}) {
|
|
212
218
|
const inheritedEnv = { ...processEnv };
|
|
213
219
|
const templateContext = buildTemplateContext(config, lease);
|
|
220
|
+
const serviceEnv = options.omitRuntimeDatabaseBindings
|
|
221
|
+
? omitRuntimeDatabaseBindings(config.testkit.serviceEnv || {})
|
|
222
|
+
: config.testkit.serviceEnv || {};
|
|
223
|
+
const localEnv = options.omitRuntimeDatabaseBindings
|
|
224
|
+
? omitRuntimeDatabaseBindings(config.testkit.local?.env || {})
|
|
225
|
+
: config.testkit.local?.env || {};
|
|
214
226
|
const env = {
|
|
215
227
|
...inheritedEnv,
|
|
216
|
-
...resolveEnvTemplates(
|
|
217
|
-
...resolveEnvTemplates(
|
|
228
|
+
...resolveEnvTemplates(serviceEnv, templateContext),
|
|
229
|
+
...resolveEnvTemplates(localEnv, templateContext),
|
|
218
230
|
...resolveEnvTemplates(extraEnv, templateContext),
|
|
219
231
|
TESTKIT_ACTIVE: "1",
|
|
220
232
|
...(config.runtimeId ? { TESTKIT_RUNTIME_ID: String(config.runtimeId) } : {}),
|
|
@@ -340,6 +352,15 @@ function resolveEnvTemplates(values, templateContext) {
|
|
|
340
352
|
);
|
|
341
353
|
}
|
|
342
354
|
|
|
355
|
+
function omitRuntimeDatabaseBindings(values = {}) {
|
|
356
|
+
return Object.fromEntries(
|
|
357
|
+
Object.entries(values).filter(([_key, value]) => {
|
|
358
|
+
if (typeof value !== "string") return true;
|
|
359
|
+
return !/\{db(?:Url|Host|Port|Name|User|Password)(?::[a-zA-Z0-9_-]+)?\}/.test(value);
|
|
360
|
+
})
|
|
361
|
+
);
|
|
362
|
+
}
|
|
363
|
+
|
|
343
364
|
function finalizeRuntimePrepare(prepare, context) {
|
|
344
365
|
if (!prepare) {
|
|
345
366
|
return {
|
|
@@ -6,25 +6,7 @@ import {
|
|
|
6
6
|
expectStatus,
|
|
7
7
|
expectStatusOneOf,
|
|
8
8
|
} from "./http-assertions.js";
|
|
9
|
-
|
|
10
|
-
const DEFAULT_PAGINATION_CASES = [
|
|
11
|
-
{ qs: "limit=0", label: "limit=0", expect400: false },
|
|
12
|
-
{ qs: "limit=-1", label: "limit=-1", expect400: true },
|
|
13
|
-
{ qs: "limit=999999", label: "limit=999999", expect400: false },
|
|
14
|
-
{ qs: "limit=abc", label: "limit=abc", expect400: true },
|
|
15
|
-
{ qs: "offset=-1", label: "offset=-1", expect400: true },
|
|
16
|
-
{ qs: "offset=1.5", label: "offset=1.5", expect400: true },
|
|
17
|
-
];
|
|
18
|
-
|
|
19
|
-
const AUDIT_LOGS_PAGINATION_CASES = [
|
|
20
|
-
{ qs: "limit=1e3", label: "limit=1e3 (scientific notation)" },
|
|
21
|
-
{ qs: "limit=Infinity", label: "limit=Infinity" },
|
|
22
|
-
{ qs: "limit=NaN", label: "limit=NaN" },
|
|
23
|
-
{ qs: "offset=NaN", label: "offset=NaN" },
|
|
24
|
-
{ qs: "limit=", label: "limit= (empty)" },
|
|
25
|
-
{ qs: "offset=", label: "offset= (empty)" },
|
|
26
|
-
{ qs: "limit=0x10", label: "limit=0x10 (hex)" },
|
|
27
|
-
];
|
|
9
|
+
import { buildPaginationCases, normalizeRequestCase } from "../shared/http-check-plan.mjs";
|
|
28
10
|
|
|
29
11
|
export function runAuthGateChecks(rawReq, scope, descriptors = {}) {
|
|
30
12
|
const {
|
|
@@ -45,43 +27,33 @@ export function runAuthGateChecks(rawReq, scope, descriptors = {}) {
|
|
|
45
27
|
|
|
46
28
|
export function runPaginationChecks(req, endpoint, options = {}) {
|
|
47
29
|
group(`${endpoint} — pagination abuse`, () => {
|
|
48
|
-
for (const {
|
|
49
|
-
const url = `${endpoint}?${qs}`;
|
|
30
|
+
for (const { label, expect400, url, auditOnly } of buildPaginationCases(endpoint, options)) {
|
|
50
31
|
const response = req.get(url);
|
|
51
32
|
|
|
52
|
-
expectNotStatus(response, 500, `${label} → not 500`);
|
|
33
|
+
expectNotStatus(response, 500, auditOnly ? `audit-logs ${label} → not 500` : `${label} → not 500`);
|
|
53
34
|
if (response.status === 500) {
|
|
54
|
-
expectResponse(
|
|
35
|
+
expectResponse(
|
|
36
|
+
response,
|
|
37
|
+
() => true,
|
|
38
|
+
auditOnly ? `BUG: audit-logs crashes on ${label}` : `BUG: ${endpoint} crashes on ${label}`
|
|
39
|
+
);
|
|
55
40
|
}
|
|
56
41
|
|
|
57
|
-
if (
|
|
42
|
+
if (auditOnly) {
|
|
43
|
+
expectStatusOneOf(response, [400, 200], `audit-logs ${label} → 400 (not silently accepted)`);
|
|
44
|
+
} else if (expect400) {
|
|
58
45
|
expectStatus(response, 400, `${label} → 400`);
|
|
59
46
|
if (response.status === 200) {
|
|
60
47
|
expectResponse(response, () => true, `BUG: ${endpoint} accepts ${label}`);
|
|
61
48
|
}
|
|
62
49
|
}
|
|
63
50
|
|
|
64
|
-
if (label === "limit=abc" && response.body) {
|
|
65
|
-
expectResponse(
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
return;
|
|
71
|
-
}
|
|
72
|
-
|
|
73
|
-
for (const { qs, label } of AUDIT_LOGS_PAGINATION_CASES) {
|
|
74
|
-
const url = `${endpoint}?${qs}`;
|
|
75
|
-
const response = req.get(url);
|
|
76
|
-
|
|
77
|
-
expectNotStatus(response, 500, `audit-logs ${label} → not 500`);
|
|
78
|
-
if (response.status === 500) {
|
|
79
|
-
expectResponse(response, () => true, `BUG: audit-logs crashes on ${label}`);
|
|
80
|
-
}
|
|
81
|
-
|
|
82
|
-
expectStatusOneOf(response, [400, 200], `audit-logs ${label} → 400 (not silently accepted)`);
|
|
83
|
-
if (response.status === 200 && response.body) {
|
|
84
|
-
expectResponse(response, (value) => !value.body.includes("NaN"), `audit-logs ${label} → no NaN in response`);
|
|
51
|
+
if ((auditOnly || label === "limit=abc") && response.status === 200 && response.body) {
|
|
52
|
+
expectResponse(
|
|
53
|
+
response,
|
|
54
|
+
(value) => !value.body.includes("NaN"),
|
|
55
|
+
auditOnly ? `audit-logs ${label} → no NaN in response` : `${label} → no NaN in response`
|
|
56
|
+
);
|
|
85
57
|
}
|
|
86
58
|
}
|
|
87
59
|
});
|
|
@@ -104,17 +76,3 @@ function runMethodAuthGateChecks(rawReq, scope, method, cases, validateErrorShap
|
|
|
104
76
|
}
|
|
105
77
|
});
|
|
106
78
|
}
|
|
107
|
-
|
|
108
|
-
function normalizeRequestCase(entry) {
|
|
109
|
-
if (Array.isArray(entry)) {
|
|
110
|
-
return {
|
|
111
|
-
path: entry[0],
|
|
112
|
-
body: entry[1],
|
|
113
|
-
};
|
|
114
|
-
}
|
|
115
|
-
|
|
116
|
-
return {
|
|
117
|
-
path: entry,
|
|
118
|
-
body: undefined,
|
|
119
|
-
};
|
|
120
|
-
}
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
export const DEFAULT_PAGINATION_CASES = [
|
|
2
|
+
{ qs: "limit=0", label: "limit=0", expect400: false },
|
|
3
|
+
{ qs: "limit=-1", label: "limit=-1", expect400: true },
|
|
4
|
+
{ qs: "limit=999999", label: "limit=999999", expect400: false },
|
|
5
|
+
{ qs: "limit=abc", label: "limit=abc", expect400: true },
|
|
6
|
+
{ qs: "offset=-1", label: "offset=-1", expect400: true },
|
|
7
|
+
{ qs: "offset=1.5", label: "offset=1.5", expect400: true },
|
|
8
|
+
];
|
|
9
|
+
|
|
10
|
+
export const AUDIT_LOGS_PAGINATION_CASES = [
|
|
11
|
+
{ qs: "limit=1e3", label: "limit=1e3 (scientific notation)" },
|
|
12
|
+
{ qs: "limit=Infinity", label: "limit=Infinity" },
|
|
13
|
+
{ qs: "limit=NaN", label: "limit=NaN" },
|
|
14
|
+
{ qs: "offset=NaN", label: "offset=NaN" },
|
|
15
|
+
{ qs: "limit=", label: "limit= (empty)" },
|
|
16
|
+
{ qs: "offset=", label: "offset= (empty)" },
|
|
17
|
+
{ qs: "limit=0x10", label: "limit=0x10 (hex)" },
|
|
18
|
+
];
|
|
19
|
+
|
|
20
|
+
export function normalizeRequestCase(entry) {
|
|
21
|
+
if (Array.isArray(entry)) {
|
|
22
|
+
return {
|
|
23
|
+
path: entry[0],
|
|
24
|
+
body: entry[1],
|
|
25
|
+
};
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
return {
|
|
29
|
+
path: entry,
|
|
30
|
+
body: undefined,
|
|
31
|
+
};
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
export function buildPaginationCases(endpoint, options = {}) {
|
|
35
|
+
const cases = DEFAULT_PAGINATION_CASES.map((entry) => ({
|
|
36
|
+
...entry,
|
|
37
|
+
url: `${endpoint}?${entry.qs}`,
|
|
38
|
+
auditOnly: false,
|
|
39
|
+
}));
|
|
40
|
+
|
|
41
|
+
if (!options.auditLogsExtra) {
|
|
42
|
+
return cases;
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
return [
|
|
46
|
+
...cases,
|
|
47
|
+
...AUDIT_LOGS_PAGINATION_CASES.map((entry) => ({
|
|
48
|
+
...entry,
|
|
49
|
+
url: `${endpoint}?${entry.qs}`,
|
|
50
|
+
auditOnly: true,
|
|
51
|
+
})),
|
|
52
|
+
];
|
|
53
|
+
}
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@elench/testkit-bridge",
|
|
3
|
-
"version": "0.1.
|
|
3
|
+
"version": "0.1.95",
|
|
4
4
|
"description": "Browser bridge helpers for testkit",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "./dist/index.js",
|
|
@@ -22,7 +22,7 @@
|
|
|
22
22
|
"typecheck": "tsc -p tsconfig.json --noEmit"
|
|
23
23
|
},
|
|
24
24
|
"dependencies": {
|
|
25
|
-
"@elench/testkit-protocol": "0.1.
|
|
25
|
+
"@elench/testkit-protocol": "0.1.95"
|
|
26
26
|
},
|
|
27
27
|
"private": false
|
|
28
28
|
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@elench/testkit",
|
|
3
|
-
"version": "0.1.
|
|
3
|
+
"version": "0.1.95",
|
|
4
4
|
"description": "Assistant-first CLI for running, inspecting, and debugging local testkit suites",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"workspaces": [
|
|
@@ -59,7 +59,8 @@
|
|
|
59
59
|
"build:packages": "npm --workspace packages/testkit-protocol run build && npm --workspace packages/ts-analysis run build && npm --workspace packages/next-analysis run build && npm --workspace packages/testkit-bridge run build",
|
|
60
60
|
"typecheck:packages": "npm --workspace packages/testkit-protocol run typecheck && npm --workspace packages/ts-analysis run typecheck && npm --workspace packages/next-analysis run typecheck && npm --workspace packages/testkit-bridge run typecheck && npm --workspace packages/testkit-extension run compile",
|
|
61
61
|
"test": "npm run build:packages && vitest run",
|
|
62
|
-
"test:
|
|
62
|
+
"test:audit": "node scripts/test-boundary-audit.mjs",
|
|
63
|
+
"test:unit": "npm run build:packages && npm run test:audit && vitest run --config vitest.unit.config.mjs",
|
|
63
64
|
"test:integration": "npm run build:packages && vitest run test/integration",
|
|
64
65
|
"test:system": "npm run build:packages && vitest run test/system --passWithNoTests"
|
|
65
66
|
},
|
|
@@ -82,10 +83,10 @@
|
|
|
82
83
|
},
|
|
83
84
|
"dependencies": {
|
|
84
85
|
"@babel/code-frame": "^7.29.0",
|
|
85
|
-
"@elench/next-analysis": "0.1.
|
|
86
|
-
"@elench/testkit-bridge": "0.1.
|
|
87
|
-
"@elench/testkit-protocol": "0.1.
|
|
88
|
-
"@elench/ts-analysis": "0.1.
|
|
86
|
+
"@elench/next-analysis": "0.1.95",
|
|
87
|
+
"@elench/testkit-bridge": "0.1.95",
|
|
88
|
+
"@elench/testkit-protocol": "0.1.95",
|
|
89
|
+
"@elench/ts-analysis": "0.1.95",
|
|
89
90
|
"@oclif/core": "^4.10.6",
|
|
90
91
|
"esbuild": "^0.25.11",
|
|
91
92
|
"execa": "^9.5.0",
|