@thecolonylab/hivemind 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/bin/hivemind.js +2 -0
- package/dist/browser/session.js +21 -0
- package/dist/cli.js +78 -0
- package/dist/commands/apply.js +33 -0
- package/dist/commands/coverage.js +98 -0
- package/dist/commands/coverageLocal.js +60 -0
- package/dist/commands/create.js +40 -0
- package/dist/commands/dashboard.js +12 -0
- package/dist/commands/explore.js +34 -0
- package/dist/commands/init.js +50 -0
- package/dist/commands/login.js +20 -0
- package/dist/commands/remoteExecutor.js +188 -0
- package/dist/commands/report.js +44 -0
- package/dist/commands/run.js +171 -0
- package/dist/commands/runLocal.js +99 -0
- package/dist/config.js +37 -0
- package/dist/credentials.js +24 -0
- package/dist/report/artifacts.js +48 -0
- package/dist/report/html.js +299 -0
- package/dist/report/prComment.js +50 -0
- package/dist/report/terminal.js +21 -0
- package/dist/serverClient.js +57 -0
- package/dist/testFile/discover.js +19 -0
- package/dist/testFile/listExisting.js +24 -0
- package/dist/testFile/loadAllTests.js +31 -0
- package/dist/testFile/parse.js +91 -0
- package/dist/testFile/resolve.js +65 -0
- package/dist/testFile/select.js +39 -0
- package/dist/testFile/types.js +1 -0
- package/dist/testFile/write.js +37 -0
- package/dist/types.js +4 -0
- package/package.json +35 -0
- package/skill/AGENTS.md.template +17 -0
- package/skill/hivemind-cli/SKILL.md +142 -0
|
@@ -0,0 +1,299 @@
|
|
|
1
|
+
function escapeHtml(s) {
|
|
2
|
+
return s.replace(/[&<>"']/g, (c) => ({ '&': '&', '<': '<', '>': '>', '"': '"', "'": ''' }[c]));
|
|
3
|
+
}
|
|
4
|
+
function formatTime(sec) {
|
|
5
|
+
const m = Math.floor(sec / 60);
|
|
6
|
+
const s = Math.floor(sec % 60);
|
|
7
|
+
return `${m}:${String(s).padStart(2, '0')}`;
|
|
8
|
+
}
|
|
9
|
+
const TYPE_ICON = {
|
|
10
|
+
act: 'โก',
|
|
11
|
+
assert: '๐',
|
|
12
|
+
login: '๐',
|
|
13
|
+
};
|
|
14
|
+
function renderStepRow(s, isFirst) {
|
|
15
|
+
const timeBadge = s.elapsedSec != null ? `<span class="step-time">${formatTime(s.elapsedSec)}</span>` : '';
|
|
16
|
+
return `
|
|
17
|
+
<li class="step ${isFirst ? 'is-active' : ''}" data-shot="${s.screenshot ?? ''}" data-note="${escapeHtml(s.note)}" data-label="${escapeHtml(s.text)}">
|
|
18
|
+
<span class="step-num">${s.index}</span>
|
|
19
|
+
<span class="step-icon">${TYPE_ICON[s.type]}</span>
|
|
20
|
+
<span class="step-text">${escapeHtml(s.text)}</span>
|
|
21
|
+
${timeBadge}
|
|
22
|
+
<span class="step-status ${s.ok ? 'ok' : 'fail'}">${s.ok ? 'โ' : 'โ'}</span>
|
|
23
|
+
</li>`;
|
|
24
|
+
}
|
|
25
|
+
function renderTraceRow(l) {
|
|
26
|
+
const inputStr = l.input ? escapeHtml(JSON.stringify(l.input)) : '';
|
|
27
|
+
return `
|
|
28
|
+
<li class="trace-row ${l.ok ? 'ok' : 'fail'}">
|
|
29
|
+
<span class="trace-num">${l.index}</span>
|
|
30
|
+
<span class="trace-status">${l.ok ? 'โ' : 'โ'}</span>
|
|
31
|
+
<span class="trace-body">
|
|
32
|
+
<span class="trace-tool">${escapeHtml(l.tool || '(final response)')}</span>${inputStr ? `<span class="trace-input">${inputStr}</span>` : ''}
|
|
33
|
+
<span class="trace-reasoning">${escapeHtml(l.reasoning)}</span>
|
|
34
|
+
${l.result ? `<span class="trace-result">${escapeHtml(l.result)}</span>` : ''}
|
|
35
|
+
</span>
|
|
36
|
+
</li>`;
|
|
37
|
+
}
|
|
38
|
+
export function renderHtmlReport(testName, result) {
|
|
39
|
+
const passed = result.status === 'passed';
|
|
40
|
+
const statusWord = passed ? 'PASS' : result.status === 'error' ? 'ERROR' : 'FAIL';
|
|
41
|
+
const okCount = result.steps.filter((s) => s.ok).length;
|
|
42
|
+
const hasShots = result.screenshots.length > 0;
|
|
43
|
+
// No video โ the evidence is the run's screenshots in order, played back as a slideshow (see
|
|
44
|
+
// the Play button below) or browsed one at a time. Inline base64 data: URIs, no files on disk.
|
|
45
|
+
const dataUris = result.screenshots.map((sc) => `data:image/png;base64,${sc.data}`);
|
|
46
|
+
const firstIndex = result.steps.find((s) => s.screenshot != null)?.screenshot ?? (hasShots ? 0 : undefined);
|
|
47
|
+
const firstShot = firstIndex != null ? dataUris[firstIndex] : '';
|
|
48
|
+
const firstLabel = result.steps.find((s) => s.screenshot != null)?.text || result.screenshots[0]?.label || '';
|
|
49
|
+
const stepsHtml = result.steps.map((s, i) => renderStepRow(s, i === 0)).join('\n');
|
|
50
|
+
const traceHtml = result.toolLog.map(renderTraceRow).join('\n');
|
|
51
|
+
const galleryHtml = result.screenshots
|
|
52
|
+
.map((sc, i) => `<button class="thumb" data-shot="${i}" data-label="${escapeHtml(sc.label || '')}"><img src="${dataUris[i]}" loading="lazy" /></button>`)
|
|
53
|
+
.join('\n');
|
|
54
|
+
const rightPanelHtml = `<div class="viewer" id="viewer">
|
|
55
|
+
${firstShot ? `<img id="viewer-img" src="${firstShot}" />` : '<div class="empty">No screenshots for this run</div>'}
|
|
56
|
+
</div>
|
|
57
|
+
${hasShots
|
|
58
|
+
? `<button id="play-btn" class="play-btn" title="Play through every screenshot like a video">โถ Play</button>`
|
|
59
|
+
: ''}
|
|
60
|
+
<div class="caption" id="viewer-caption">${escapeHtml(firstLabel)}</div>`;
|
|
61
|
+
return `<!doctype html>
|
|
62
|
+
<html lang="en">
|
|
63
|
+
<head>
|
|
64
|
+
<meta charset="utf-8" />
|
|
65
|
+
<meta name="viewport" content="width=device-width, initial-scale=1" />
|
|
66
|
+
<title>${escapeHtml(testName)} ยท hivemind</title>
|
|
67
|
+
<style>
|
|
68
|
+
:root {
|
|
69
|
+
color-scheme: dark;
|
|
70
|
+
--bg: #0a0b0d;
|
|
71
|
+
--panel: #111318;
|
|
72
|
+
--panel-2: #171a20;
|
|
73
|
+
--border: #24272e;
|
|
74
|
+
--text: #eceef1;
|
|
75
|
+
--text-dim: #8a8f98;
|
|
76
|
+
--text-faint: #565b64;
|
|
77
|
+
--pass: #22c55e;
|
|
78
|
+
--fail: #f43f5e;
|
|
79
|
+
--accent: #6ea8fe;
|
|
80
|
+
}
|
|
81
|
+
* { box-sizing: border-box; }
|
|
82
|
+
html, body { margin: 0; height: 100%; background: var(--bg); }
|
|
83
|
+
body {
|
|
84
|
+
color: var(--text);
|
|
85
|
+
font-family: -apple-system, BlinkMacSystemFont, "Inter", "Segoe UI", sans-serif;
|
|
86
|
+
-webkit-font-smoothing: antialiased;
|
|
87
|
+
display: flex; flex-direction: column;
|
|
88
|
+
min-height: 100vh;
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
.topbar {
|
|
92
|
+
display: flex; align-items: center; gap: 12px;
|
|
93
|
+
padding: 14px 20px; border-bottom: 1px solid var(--border);
|
|
94
|
+
background: var(--panel);
|
|
95
|
+
}
|
|
96
|
+
.topbar h1 { font-size: 15px; font-weight: 600; margin: 0; letter-spacing: -0.01em; }
|
|
97
|
+
.topbar .badge {
|
|
98
|
+
font-size: 11px; font-weight: 700; letter-spacing: 0.04em; padding: 3px 9px; border-radius: 5px;
|
|
99
|
+
color: #05130a; background: ${passed ? 'var(--pass)' : 'var(--fail)'};
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
.layout { display: flex; flex: 1; min-height: 0; }
|
|
103
|
+
|
|
104
|
+
.left {
|
|
105
|
+
width: 420px; flex-shrink: 0; display: flex; flex-direction: column;
|
|
106
|
+
border-right: 1px solid var(--border); background: var(--bg);
|
|
107
|
+
}
|
|
108
|
+
ul.steps { list-style: none; margin: 0; padding: 6px; overflow-y: auto; flex: 1; }
|
|
109
|
+
.step {
|
|
110
|
+
display: flex; align-items: center; gap: 8px;
|
|
111
|
+
padding: 10px 10px; border-radius: 8px; cursor: pointer;
|
|
112
|
+
border: 1px solid transparent;
|
|
113
|
+
}
|
|
114
|
+
.step:hover { background: var(--panel-2); }
|
|
115
|
+
.step.is-active { background: var(--panel-2); border-color: var(--border); }
|
|
116
|
+
.step-num { font-size: 11px; color: var(--text-faint); width: 16px; text-align: right; flex-shrink: 0; }
|
|
117
|
+
.step-icon { font-size: 13px; flex-shrink: 0; opacity: 0.9; }
|
|
118
|
+
.step-text { font-size: 13.5px; color: var(--text); flex: 1; line-height: 1.4; }
|
|
119
|
+
.step-time {
|
|
120
|
+
font-size: 11px; color: var(--accent); flex-shrink: 0; white-space: nowrap;
|
|
121
|
+
background: var(--panel-2); border: 1px solid var(--border); border-radius: 999px; padding: 2px 8px;
|
|
122
|
+
}
|
|
123
|
+
.step-time:hover { border-color: var(--accent); }
|
|
124
|
+
.step-status { font-size: 12px; font-weight: 700; flex-shrink: 0; }
|
|
125
|
+
.step-status.ok { color: var(--pass); }
|
|
126
|
+
.step-status.fail { color: var(--fail); }
|
|
127
|
+
|
|
128
|
+
.summary-card {
|
|
129
|
+
margin: 10px; padding: 14px 16px; border-radius: 10px;
|
|
130
|
+
background: var(--panel); border: 1px solid var(--border);
|
|
131
|
+
}
|
|
132
|
+
.summary-top { display: flex; align-items: center; justify-content: space-between; margin-bottom: 8px; }
|
|
133
|
+
.summary-name { font-size: 13px; font-weight: 600; }
|
|
134
|
+
.summary-meta { display: flex; align-items: center; gap: 8px; }
|
|
135
|
+
.summary-time { font-size: 11px; color: var(--text-faint); }
|
|
136
|
+
.summary-badge { font-size: 10.5px; font-weight: 700; padding: 2px 8px; border-radius: 5px; color: #05130a; background: ${passed ? 'var(--pass)' : 'var(--fail)'}; }
|
|
137
|
+
.summary-verdict { font-size: 12.5px; color: var(--text-dim); margin-bottom: 10px; }
|
|
138
|
+
.summary-row { font-size: 12px; color: var(--text-dim); display: flex; justify-content: space-between; }
|
|
139
|
+
|
|
140
|
+
.right {
|
|
141
|
+
flex: 1; display: flex; flex-direction: column; background: #000;
|
|
142
|
+
align-items: center; justify-content: center; position: relative; min-width: 0;
|
|
143
|
+
}
|
|
144
|
+
.viewer { width: 100%; height: 100%; display: flex; align-items: center; justify-content: center; padding: 24px; }
|
|
145
|
+
.viewer img { max-width: 100%; max-height: 100%; border-radius: 6px; box-shadow: 0 0 0 1px var(--border); object-fit: contain; }
|
|
146
|
+
.viewer .empty { color: var(--text-faint); font-size: 13px; }
|
|
147
|
+
.play-btn {
|
|
148
|
+
position: absolute; top: 16px; right: 16px;
|
|
149
|
+
display: flex; align-items: center; gap: 6px;
|
|
150
|
+
padding: 7px 14px; border-radius: 999px; border: 1px solid var(--border);
|
|
151
|
+
background: rgba(17,19,24,0.85); color: var(--text); font-size: 12.5px; font-weight: 600;
|
|
152
|
+
cursor: pointer; backdrop-filter: blur(4px);
|
|
153
|
+
}
|
|
154
|
+
.play-btn:hover { border-color: var(--accent); color: var(--accent); }
|
|
155
|
+
.play-btn.is-playing { border-color: var(--accent); color: var(--accent); }
|
|
156
|
+
.caption {
|
|
157
|
+
position: absolute; bottom: 0; left: 0; right: 0;
|
|
158
|
+
padding: 10px 20px; font-size: 12px; color: var(--text-dim);
|
|
159
|
+
background: linear-gradient(0deg, #000 0%, transparent 100%);
|
|
160
|
+
}
|
|
161
|
+
|
|
162
|
+
.gallery-strip { display: flex; gap: 8px; padding: 10px; overflow-x: auto; border-top: 1px solid var(--border); }
|
|
163
|
+
.thumb { padding: 0; border: 1px solid var(--border); border-radius: 6px; overflow: hidden; background: none; cursor: pointer; flex-shrink: 0; width: 90px; height: 60px; }
|
|
164
|
+
.thumb img { width: 100%; height: 100%; object-fit: cover; display: block; }
|
|
165
|
+
.thumb:hover { border-color: var(--accent); }
|
|
166
|
+
|
|
167
|
+
.trace { border-top: 1px solid var(--border); background: var(--panel); }
|
|
168
|
+
.trace summary {
|
|
169
|
+
padding: 10px 20px; font-size: 12px; color: var(--text-dim); cursor: pointer;
|
|
170
|
+
list-style: none; user-select: none;
|
|
171
|
+
}
|
|
172
|
+
.trace summary::-webkit-details-marker { display: none; }
|
|
173
|
+
.trace summary::before { content: 'โถ '; font-size: 9px; }
|
|
174
|
+
.trace[open] summary::before { content: 'โผ '; }
|
|
175
|
+
ul.trace-list { list-style: none; margin: 0; padding: 0 20px 14px; max-height: 320px; overflow-y: auto; }
|
|
176
|
+
.trace-row { display: flex; gap: 10px; padding: 7px 0; border-bottom: 1px solid var(--border); font-size: 12px; }
|
|
177
|
+
.trace-row:last-child { border-bottom: none; }
|
|
178
|
+
.trace-num { color: var(--text-faint); width: 20px; flex-shrink: 0; text-align: right; }
|
|
179
|
+
.trace-status { flex-shrink: 0; font-weight: 700; }
|
|
180
|
+
.trace-row.ok .trace-status { color: var(--pass); }
|
|
181
|
+
.trace-row.fail .trace-status { color: var(--fail); }
|
|
182
|
+
.trace-body { display: flex; flex-direction: column; gap: 2px; min-width: 0; }
|
|
183
|
+
.trace-tool { color: var(--accent); font-family: ui-monospace, monospace; font-size: 11.5px; }
|
|
184
|
+
.trace-input { color: var(--text-faint); font-family: ui-monospace, monospace; font-size: 11px; margin-left: 6px; }
|
|
185
|
+
.trace-reasoning { color: var(--text-dim); }
|
|
186
|
+
.trace-result { color: var(--text-faint); font-size: 11.5px; }
|
|
187
|
+
|
|
188
|
+
@media (max-width: 860px) {
|
|
189
|
+
.layout { flex-direction: column; }
|
|
190
|
+
.left { width: 100%; border-right: none; border-bottom: 1px solid var(--border); }
|
|
191
|
+
ul.steps { max-height: 260px; }
|
|
192
|
+
}
|
|
193
|
+
</style>
|
|
194
|
+
</head>
|
|
195
|
+
<body>
|
|
196
|
+
<div class="topbar">
|
|
197
|
+
<span class="badge">${statusWord}</span>
|
|
198
|
+
<h1>${escapeHtml(testName)}</h1>
|
|
199
|
+
</div>
|
|
200
|
+
|
|
201
|
+
<div class="layout">
|
|
202
|
+
<div class="left">
|
|
203
|
+
<ul class="steps">${stepsHtml}</ul>
|
|
204
|
+
<div class="summary-card">
|
|
205
|
+
<div class="summary-top">
|
|
206
|
+
<span class="summary-name">${escapeHtml(testName)}</span>
|
|
207
|
+
<span class="summary-meta">
|
|
208
|
+
<span class="summary-time">${(result.durationMs / 1000).toFixed(1)}s</span>
|
|
209
|
+
<span class="summary-badge">${statusWord}</span>
|
|
210
|
+
</span>
|
|
211
|
+
</div>
|
|
212
|
+
<div class="summary-verdict">${escapeHtml(result.verdict)}</div>
|
|
213
|
+
<div class="summary-row"><span>${okCount}/${result.steps.length} steps passed</span><span>${result.screenshots.length} screenshot${result.screenshots.length === 1 ? '' : 's'}</span></div>
|
|
214
|
+
</div>
|
|
215
|
+
</div>
|
|
216
|
+
|
|
217
|
+
<div class="right">${rightPanelHtml}</div>
|
|
218
|
+
</div>
|
|
219
|
+
|
|
220
|
+
${result.screenshots.length ? `<div class="gallery-strip">${galleryHtml}</div>` : ''}
|
|
221
|
+
|
|
222
|
+
${result.toolLog.length
|
|
223
|
+
? `<details class="trace">
|
|
224
|
+
<summary>Agent trace (${result.toolLog.length} tool call${result.toolLog.length === 1 ? '' : 's'}) โ for debugging, not what's evaluated above</summary>
|
|
225
|
+
<ul class="trace-list">${traceHtml}</ul>
|
|
226
|
+
</details>`
|
|
227
|
+
: ''}
|
|
228
|
+
|
|
229
|
+
<script>
|
|
230
|
+
const SHOTS = ${JSON.stringify(dataUris)};
|
|
231
|
+
const LABELS = ${JSON.stringify(result.screenshots.map((s) => s.label || ''))};
|
|
232
|
+
const viewer = document.getElementById('viewer');
|
|
233
|
+
const caption = document.getElementById('viewer-caption');
|
|
234
|
+
const playBtn = document.getElementById('play-btn');
|
|
235
|
+
|
|
236
|
+
let playTimer = null;
|
|
237
|
+
|
|
238
|
+
function showShot(index, label) {
|
|
239
|
+
const src = SHOTS[index];
|
|
240
|
+
if (!src || !viewer) return;
|
|
241
|
+
viewer.innerHTML = '<img id="viewer-img" src="' + src + '" />';
|
|
242
|
+
if (caption) caption.textContent = label != null ? label : (LABELS[index] || '');
|
|
243
|
+
|
|
244
|
+
document.querySelectorAll('.step').forEach((s) => {
|
|
245
|
+
s.classList.toggle('is-active', s.getAttribute('data-shot') === String(index));
|
|
246
|
+
});
|
|
247
|
+
}
|
|
248
|
+
|
|
249
|
+
function stopPlaying() {
|
|
250
|
+
if (playTimer) clearInterval(playTimer);
|
|
251
|
+
playTimer = null;
|
|
252
|
+
if (playBtn) {
|
|
253
|
+
playBtn.textContent = 'โถ Play';
|
|
254
|
+
playBtn.classList.remove('is-playing');
|
|
255
|
+
}
|
|
256
|
+
}
|
|
257
|
+
|
|
258
|
+
function startPlaying() {
|
|
259
|
+
if (!SHOTS.length) return;
|
|
260
|
+
let i = 0;
|
|
261
|
+
showShot(i);
|
|
262
|
+
if (playBtn) {
|
|
263
|
+
playBtn.textContent = 'โโ Pause';
|
|
264
|
+
playBtn.classList.add('is-playing');
|
|
265
|
+
}
|
|
266
|
+
playTimer = setInterval(() => {
|
|
267
|
+
i++;
|
|
268
|
+
if (i >= SHOTS.length) {
|
|
269
|
+
stopPlaying();
|
|
270
|
+
return;
|
|
271
|
+
}
|
|
272
|
+
showShot(i);
|
|
273
|
+
}, 900);
|
|
274
|
+
}
|
|
275
|
+
|
|
276
|
+
if (playBtn) {
|
|
277
|
+
playBtn.addEventListener('click', () => {
|
|
278
|
+
if (playTimer) stopPlaying();
|
|
279
|
+
else startPlaying();
|
|
280
|
+
});
|
|
281
|
+
}
|
|
282
|
+
|
|
283
|
+
document.querySelectorAll('.step').forEach((el) => {
|
|
284
|
+
el.addEventListener('click', () => {
|
|
285
|
+
stopPlaying();
|
|
286
|
+
const shot = el.getAttribute('data-shot');
|
|
287
|
+
if (shot !== '') showShot(Number(shot), el.getAttribute('data-label'));
|
|
288
|
+
});
|
|
289
|
+
});
|
|
290
|
+
document.querySelectorAll('.thumb').forEach((el) => {
|
|
291
|
+
el.addEventListener('click', () => {
|
|
292
|
+
stopPlaying();
|
|
293
|
+
showShot(Number(el.getAttribute('data-shot')), el.getAttribute('data-label'));
|
|
294
|
+
});
|
|
295
|
+
});
|
|
296
|
+
</script>
|
|
297
|
+
</body>
|
|
298
|
+
</html>`;
|
|
299
|
+
}
|
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
/** Formats `hivemind run` results as a PR comment โ same numbered pass/fail list + report link
|
|
2
|
+
* shape as tester.army's own PR comment bot. Meant to be written to a file (--pr-comment <path>)
|
|
3
|
+
* and posted with `gh pr comment --body-file`. */
|
|
4
|
+
export function renderRunPrComment(results) {
|
|
5
|
+
const failed = results.filter((r) => r.result.status !== 'passed');
|
|
6
|
+
const headline = failed.length === 0 ? 'All tests passed.' : `${failed.length} of ${results.length} test${results.length === 1 ? '' : 's'} failed.`;
|
|
7
|
+
const list = results
|
|
8
|
+
.map((r, i) => {
|
|
9
|
+
const icon = r.result.status === 'passed' ? 'โ
' : r.result.status === 'error' ? 'โ ๏ธ' : 'โ';
|
|
10
|
+
return `${i + 1}. ${icon} ${escapeMd(r.name)} ${r.result.status}${r.result.status !== 'passed' ? ` โ ${escapeMd(r.result.verdict)}` : ''}`;
|
|
11
|
+
})
|
|
12
|
+
.join('\n');
|
|
13
|
+
const reportLinks = results
|
|
14
|
+
.filter((r) => r.reportUrl)
|
|
15
|
+
.map((r) => `[${escapeMd(r.name)} report](${r.reportUrl})`)
|
|
16
|
+
.join(' ยท ');
|
|
17
|
+
return [`### ๐ hivemind`, '', `**${headline}**`, '', list, '', reportLinks].filter((l) => l !== '').join('\n');
|
|
18
|
+
}
|
|
19
|
+
/** Formats `hivemind coverage` results โ for each affected page: the existing tests that were
|
|
20
|
+
* run against it (the regression check) and any newly drafted test for a gap that's left. */
|
|
21
|
+
export function renderCoveragePrComment(affectedPages) {
|
|
22
|
+
const allTestsRun = affectedPages.flatMap((p) => p.testsRun);
|
|
23
|
+
const regressions = allTestsRun.filter((t) => t.status !== 'passed');
|
|
24
|
+
const gaps = affectedPages.filter((p) => p.proposedTest);
|
|
25
|
+
const headline = regressions.length === 0
|
|
26
|
+
? gaps.length === 0
|
|
27
|
+
? 'Coverage check passed โ everything this PR touches is already tested.'
|
|
28
|
+
: `Coverage check passed, ${gaps.length} new test${gaps.length === 1 ? '' : 's'} drafted for uncovered changes.`
|
|
29
|
+
: `${regressions.length} existing test${regressions.length === 1 ? '' : 's'} broke because of this PR.`;
|
|
30
|
+
const sections = affectedPages.map((page) => {
|
|
31
|
+
const lines = [`**${escapeMd(page.url)}**`, `_${escapeMd(page.reason)}_`];
|
|
32
|
+
page.testsRun.forEach((t, i) => {
|
|
33
|
+
const icon = t.status === 'passed' ? 'โ
' : t.status === 'error' ? 'โ ๏ธ' : 'โ';
|
|
34
|
+
const reportLink = t.reportUrl ? ` โ [view report](${t.reportUrl})` : '';
|
|
35
|
+
lines.push(`${i + 1}. ${icon} ${escapeMd(t.name)} ${t.status}${t.status !== 'passed' ? ` โ ${escapeMd(t.verdict)}` : ''}${reportLink}`);
|
|
36
|
+
});
|
|
37
|
+
if (page.proposedTest) {
|
|
38
|
+
lines.push(`+ ๐ drafted **${escapeMd(page.proposedTest.name)}** for a gap this PR introduced โ review before merging`);
|
|
39
|
+
lines.push(` Run \`hivemind apply ${page.proposedTest.code}\` to pull it into your project.`);
|
|
40
|
+
}
|
|
41
|
+
if (!page.testsRun.length && !page.proposedTest) {
|
|
42
|
+
lines.push(`(no test targets this page, and nothing new stood out to write one for)`);
|
|
43
|
+
}
|
|
44
|
+
return lines.join('\n');
|
|
45
|
+
});
|
|
46
|
+
return [`### ๐ hivemind coverage`, '', `**${headline}**`, '', sections.join('\n\n')].join('\n');
|
|
47
|
+
}
|
|
48
|
+
function escapeMd(s) {
|
|
49
|
+
return s.replace(/[|_*`]/g, '\\$&');
|
|
50
|
+
}
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
const green = (s) => `\x1b[32m${s}\x1b[0m`;
|
|
2
|
+
const red = (s) => `\x1b[31m${s}\x1b[0m`;
|
|
3
|
+
const dim = (s) => `\x1b[2m${s}\x1b[0m`;
|
|
4
|
+
const bold = (s) => `\x1b[1m${s}\x1b[0m`;
|
|
5
|
+
export function printResult(testName, result) {
|
|
6
|
+
const badge = result.status === 'passed' ? green('PASS') : red(result.status === 'error' ? 'ERROR' : 'FAIL');
|
|
7
|
+
console.log(`\n${bold(testName)} ${badge} ${dim(`(${(result.durationMs / 1000).toFixed(1)}s)`)}`);
|
|
8
|
+
console.log(dim(result.verdict));
|
|
9
|
+
for (const step of result.steps) {
|
|
10
|
+
const mark = step.ok ? green('โ') : red('โ');
|
|
11
|
+
console.log(` ${mark} [${step.type}] ${step.text} ${dim('โ ' + step.note)}`);
|
|
12
|
+
}
|
|
13
|
+
if (result.screenshots.length) {
|
|
14
|
+
console.log(dim(` ${result.screenshots.length} screenshot(s) saved`));
|
|
15
|
+
}
|
|
16
|
+
}
|
|
17
|
+
export function printSummary(results) {
|
|
18
|
+
const passed = results.filter((r) => r.result.status === 'passed').length;
|
|
19
|
+
const failed = results.length - passed;
|
|
20
|
+
console.log(`\n${bold('Summary:')} ${green(`${passed} passed`)}, ${failed > 0 ? red(`${failed} failed`) : `${failed} failed`} (${results.length} total)`);
|
|
21
|
+
}
|
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
import { readApiKey } from './credentials.js';
|
|
2
|
+
export function serverUrl() {
|
|
3
|
+
return process.env.HIVEMIND_SERVER_URL || 'https://app.thecolonylab.com';
|
|
4
|
+
}
|
|
5
|
+
// Identifies every request as coming from the Forager (the CLI's role โ runs on your machine,
|
|
6
|
+
// executes exactly what the Hivemind brain tells it to) โ shows up in server-side logs as
|
|
7
|
+
// e.g. "Hivemind-Forager/0.1.0", no functional effect, just a trace of who's calling.
|
|
8
|
+
const FORAGER_USER_AGENT = 'Hivemind-Forager/0.1.0';
|
|
9
|
+
async function post(path, body) {
|
|
10
|
+
const apiKey = await readApiKey();
|
|
11
|
+
const res = await fetch(`${serverUrl()}${path}`, {
|
|
12
|
+
method: 'POST',
|
|
13
|
+
headers: {
|
|
14
|
+
'Content-Type': 'application/json',
|
|
15
|
+
'User-Agent': FORAGER_USER_AGENT,
|
|
16
|
+
...(apiKey ? { Authorization: `Bearer ${apiKey}` } : {}),
|
|
17
|
+
},
|
|
18
|
+
body: JSON.stringify(body),
|
|
19
|
+
});
|
|
20
|
+
const data = await res.json().catch(() => ({}));
|
|
21
|
+
if (!res.ok) {
|
|
22
|
+
if (res.status === 401) {
|
|
23
|
+
throw new Error(`Not signed in โ run "hivemind login <key>" (get a key from the dashboard's Settings page).`);
|
|
24
|
+
}
|
|
25
|
+
throw new Error(data?.error || `hivemind server at ${serverUrl()} returned ${res.status}. Is it running and reachable?`);
|
|
26
|
+
}
|
|
27
|
+
return data;
|
|
28
|
+
}
|
|
29
|
+
export async function runOnServer(input) {
|
|
30
|
+
return post('/api/runs', input);
|
|
31
|
+
}
|
|
32
|
+
export async function createOnServer(request, url, existingTests, accountLabels) {
|
|
33
|
+
return post('/api/create', { request, url, existingTests, accountLabels });
|
|
34
|
+
}
|
|
35
|
+
export async function exploreOnServer(url, existingTests) {
|
|
36
|
+
return post('/api/explore', { url, existingTests });
|
|
37
|
+
}
|
|
38
|
+
export async function coverageOnServer(diff, localTests) {
|
|
39
|
+
return post('/api/coverage', { diff, localTests });
|
|
40
|
+
}
|
|
41
|
+
export async function getDraftTest(code) {
|
|
42
|
+
const apiKey = await readApiKey();
|
|
43
|
+
const res = await fetch(`${serverUrl()}/api/coverage/draft/${encodeURIComponent(code)}`, {
|
|
44
|
+
headers: {
|
|
45
|
+
'User-Agent': FORAGER_USER_AGENT,
|
|
46
|
+
...(apiKey ? { Authorization: `Bearer ${apiKey}` } : {}),
|
|
47
|
+
},
|
|
48
|
+
});
|
|
49
|
+
const data = await res.json().catch(() => ({}));
|
|
50
|
+
if (!res.ok) {
|
|
51
|
+
if (res.status === 401) {
|
|
52
|
+
throw new Error(`Not signed in โ run "hivemind login <key>" (get a key from the dashboard's Settings page).`);
|
|
53
|
+
}
|
|
54
|
+
throw new Error(data?.error || `hivemind server at ${serverUrl()} returned ${res.status}. Is it running and reachable?`);
|
|
55
|
+
}
|
|
56
|
+
return data;
|
|
57
|
+
}
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
import { readdir, stat } from 'node:fs/promises';
|
|
2
|
+
import { join } from 'node:path';
|
|
3
|
+
export async function findTestFiles(target) {
|
|
4
|
+
const s = await stat(target);
|
|
5
|
+
if (s.isFile())
|
|
6
|
+
return [target];
|
|
7
|
+
const out = [];
|
|
8
|
+
async function walk(dir) {
|
|
9
|
+
for (const entry of await readdir(dir, { withFileTypes: true })) {
|
|
10
|
+
const full = join(dir, entry.name);
|
|
11
|
+
if (entry.isDirectory())
|
|
12
|
+
await walk(full);
|
|
13
|
+
else if (entry.name.endsWith('.md'))
|
|
14
|
+
out.push(full);
|
|
15
|
+
}
|
|
16
|
+
}
|
|
17
|
+
await walk(target);
|
|
18
|
+
return out.sort();
|
|
19
|
+
}
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
import { join, basename } from 'node:path';
|
|
2
|
+
import { stat } from 'node:fs/promises';
|
|
3
|
+
import { findTestFiles } from './discover.js';
|
|
4
|
+
import { loadTestFile } from './parse.js';
|
|
5
|
+
/** Scans hive/tests/ so `create` can tell the AI what already exists โ letting it reference an
|
|
6
|
+
* existing test via Before: instead of re-describing the same setup steps every time. */
|
|
7
|
+
export async function listExistingTests(cwd) {
|
|
8
|
+
const testsDir = join(cwd, 'hive', 'tests');
|
|
9
|
+
try {
|
|
10
|
+
await stat(testsDir);
|
|
11
|
+
}
|
|
12
|
+
catch {
|
|
13
|
+
return [];
|
|
14
|
+
}
|
|
15
|
+
const files = await findTestFiles(testsDir);
|
|
16
|
+
const summaries = await Promise.all(files.map(async (f) => {
|
|
17
|
+
const test = await loadTestFile(f);
|
|
18
|
+
// Matches resolve.ts's lookup exactly: Before: references match by basename anywhere
|
|
19
|
+
// in the tree, not by relative path โ so this must be the same key space.
|
|
20
|
+
const file = basename(f).replace(/\.md$/, '');
|
|
21
|
+
return { file, name: test.name, tags: test.tags, params: Object.keys(test.params) };
|
|
22
|
+
}));
|
|
23
|
+
return summaries;
|
|
24
|
+
}
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
import { join } from 'node:path';
|
|
2
|
+
import { findTestFiles } from './discover.js';
|
|
3
|
+
import { loadTestFile } from './parse.js';
|
|
4
|
+
import { resolveSteps } from './resolve.js';
|
|
5
|
+
/** Loads every local test with its fully-resolved steps (Before:/Params: already expanded) โ
|
|
6
|
+
* not just the {file, name, tags} summary `create`/`explore` use for dedup prompts. Coverage
|
|
7
|
+
* needs the real steps because its whole job is to actually RUN whichever of these tests match
|
|
8
|
+
* an affected page, not just know they exist. */
|
|
9
|
+
export async function loadAllTestsFull(cwd) {
|
|
10
|
+
const testsRoot = join(cwd, 'hive', 'tests');
|
|
11
|
+
let files;
|
|
12
|
+
try {
|
|
13
|
+
files = await findTestFiles(testsRoot);
|
|
14
|
+
}
|
|
15
|
+
catch {
|
|
16
|
+
return [];
|
|
17
|
+
}
|
|
18
|
+
const out = [];
|
|
19
|
+
for (const file of files) {
|
|
20
|
+
try {
|
|
21
|
+
const test = await loadTestFile(file);
|
|
22
|
+
const steps = await resolveSteps(test, testsRoot);
|
|
23
|
+
out.push({ name: test.name, url: test.url, tags: test.tags, steps });
|
|
24
|
+
}
|
|
25
|
+
catch {
|
|
26
|
+
// A test file with a broken Before: reference or similar โ skip it rather than fail the
|
|
27
|
+
// whole coverage check over one unrelated bad file.
|
|
28
|
+
}
|
|
29
|
+
}
|
|
30
|
+
return out;
|
|
31
|
+
}
|
|
@@ -0,0 +1,91 @@
|
|
|
1
|
+
import { readFile } from 'node:fs/promises';
|
|
2
|
+
/** Splits "key=value" or "key="quoted value"" pairs out of a line, comma or space separated. */
|
|
3
|
+
function parseKeyValuePairs(text) {
|
|
4
|
+
const out = {};
|
|
5
|
+
const regex = /(\w+)\s*=\s*(?:"([^"]*)"|([^\s,]+))/g;
|
|
6
|
+
let m;
|
|
7
|
+
while ((m = regex.exec(text))) {
|
|
8
|
+
out[m[1]] = m[2] !== undefined ? m[2] : m[3];
|
|
9
|
+
}
|
|
10
|
+
return out;
|
|
11
|
+
}
|
|
12
|
+
/** Parses one `Before:` line: `<name> [param="value" ...]` */
|
|
13
|
+
function parseBeforeLine(text) {
|
|
14
|
+
const params = parseKeyValuePairs(text);
|
|
15
|
+
const paramRegex = /\w+\s*=\s*(?:"[^"]*"|[^\s,]+)/g;
|
|
16
|
+
const name = text.replace(paramRegex, '').trim();
|
|
17
|
+
return { name, params };
|
|
18
|
+
}
|
|
19
|
+
/**
|
|
20
|
+
* Parses the hivemind test markdown format:
|
|
21
|
+
*
|
|
22
|
+
* # <name>
|
|
23
|
+
* URL: <url>
|
|
24
|
+
* Login: <account label>
|
|
25
|
+
* Tags: a, b, c
|
|
26
|
+
* Params: product=any available item
|
|
27
|
+
* Before: login
|
|
28
|
+
* Before: add-item-to-cart product="Blue T-Shirt"
|
|
29
|
+
*
|
|
30
|
+
* ## Steps
|
|
31
|
+
* 1. act: click "Login"
|
|
32
|
+
* 2. assert: dashboard loads
|
|
33
|
+
*/
|
|
34
|
+
export function parseTestFile(content, filePath) {
|
|
35
|
+
const lines = content.split('\n');
|
|
36
|
+
let name = filePath ? filePath.split('/').pop().replace(/\.md$/, '') : 'Untitled test';
|
|
37
|
+
let url;
|
|
38
|
+
let login;
|
|
39
|
+
let tags = [];
|
|
40
|
+
let params = {};
|
|
41
|
+
const before = [];
|
|
42
|
+
const steps = [];
|
|
43
|
+
for (const raw of lines) {
|
|
44
|
+
const line = raw.trim();
|
|
45
|
+
if (line.startsWith('# ')) {
|
|
46
|
+
name = line.slice(2).trim();
|
|
47
|
+
continue;
|
|
48
|
+
}
|
|
49
|
+
const urlMatch = /^URL:\s*(.+)$/i.exec(line);
|
|
50
|
+
if (urlMatch) {
|
|
51
|
+
url = urlMatch[1].trim();
|
|
52
|
+
continue;
|
|
53
|
+
}
|
|
54
|
+
const loginMatch = /^Login:\s*(.+)$/i.exec(line);
|
|
55
|
+
if (loginMatch) {
|
|
56
|
+
login = loginMatch[1].trim();
|
|
57
|
+
continue;
|
|
58
|
+
}
|
|
59
|
+
const tagsMatch = /^Tags:\s*(.+)$/i.exec(line);
|
|
60
|
+
if (tagsMatch) {
|
|
61
|
+
tags = tagsMatch[1].split(',').map((t) => t.trim()).filter(Boolean);
|
|
62
|
+
continue;
|
|
63
|
+
}
|
|
64
|
+
const paramsMatch = /^Params:\s*(.+)$/i.exec(line);
|
|
65
|
+
if (paramsMatch) {
|
|
66
|
+
params = { ...params, ...parseKeyValuePairs(paramsMatch[1]) };
|
|
67
|
+
continue;
|
|
68
|
+
}
|
|
69
|
+
const beforeMatch = /^Before:\s*(.+)$/i.exec(line);
|
|
70
|
+
if (beforeMatch) {
|
|
71
|
+
before.push(parseBeforeLine(beforeMatch[1].trim()));
|
|
72
|
+
continue;
|
|
73
|
+
}
|
|
74
|
+
// Numbered typed step: "1. act: do the thing" or "1. assert: check the thing"
|
|
75
|
+
const stepMatch = /^\d+\.\s*(act|assert|login)\s*:\s*(.+)$/i.exec(line);
|
|
76
|
+
if (stepMatch) {
|
|
77
|
+
steps.push({ type: stepMatch[1].toLowerCase(), text: stepMatch[2].trim() });
|
|
78
|
+
continue;
|
|
79
|
+
}
|
|
80
|
+
// Fallback: untyped numbered step defaults to "act"
|
|
81
|
+
const plainStepMatch = /^\d+\.\s*(.+)$/.exec(line);
|
|
82
|
+
if (plainStepMatch && !line.startsWith('##')) {
|
|
83
|
+
steps.push({ type: 'act', text: plainStepMatch[1].trim() });
|
|
84
|
+
}
|
|
85
|
+
}
|
|
86
|
+
return { name, url, login, tags, params, before, steps, filePath };
|
|
87
|
+
}
|
|
88
|
+
export async function loadTestFile(filePath) {
|
|
89
|
+
const content = await readFile(filePath, 'utf-8');
|
|
90
|
+
return parseTestFile(content, filePath);
|
|
91
|
+
}
|
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
import { readdir } from 'node:fs/promises';
|
|
2
|
+
import { join } from 'node:path';
|
|
3
|
+
import { loadTestFile } from './parse.js';
|
|
4
|
+
/** Finds a test file anywhere under testsDir whose filename (minus .md) matches `name`. */
|
|
5
|
+
async function findByName(testsDir, name) {
|
|
6
|
+
const target = `${name}.md`;
|
|
7
|
+
async function walk(dir) {
|
|
8
|
+
let entries;
|
|
9
|
+
try {
|
|
10
|
+
entries = await readdir(dir, { withFileTypes: true });
|
|
11
|
+
}
|
|
12
|
+
catch {
|
|
13
|
+
return undefined;
|
|
14
|
+
}
|
|
15
|
+
for (const entry of entries) {
|
|
16
|
+
const full = join(dir, entry.name);
|
|
17
|
+
if (entry.isDirectory()) {
|
|
18
|
+
const found = await walk(full);
|
|
19
|
+
if (found)
|
|
20
|
+
return found;
|
|
21
|
+
}
|
|
22
|
+
else if (entry.name === target) {
|
|
23
|
+
return full;
|
|
24
|
+
}
|
|
25
|
+
}
|
|
26
|
+
return undefined;
|
|
27
|
+
}
|
|
28
|
+
return walk(testsDir);
|
|
29
|
+
}
|
|
30
|
+
function substitute(text, params) {
|
|
31
|
+
return text.replace(/\{\{(\w+)\}\}/g, (_, key) => params[key] ?? `{{${key}}}`);
|
|
32
|
+
}
|
|
33
|
+
/**
|
|
34
|
+
* Expands a test's Before: chain and {{param}} placeholders into one flat, ready-to-run step
|
|
35
|
+
* list. Runs entirely client-side (the CLI already has full access to hive/tests/) โ the server
|
|
36
|
+
* never needs to know Before:/Params exist at all, it just receives a plain steps[] array like
|
|
37
|
+
* it always has.
|
|
38
|
+
*/
|
|
39
|
+
export async function resolveSteps(test, testsDir) {
|
|
40
|
+
const seen = new Set();
|
|
41
|
+
async function expand(file, callParams) {
|
|
42
|
+
// Identity for cycle detection is the file path (stable, always set by loadTestFile) โ
|
|
43
|
+
// NOT file.name, since two different files could share the same human-readable title.
|
|
44
|
+
const identity = file.filePath ?? file.name;
|
|
45
|
+
if (seen.has(identity)) {
|
|
46
|
+
throw new Error(`Before: cycle detected โ "${file.name}" is referenced by something it also (indirectly) depends on.`);
|
|
47
|
+
}
|
|
48
|
+
seen.add(identity);
|
|
49
|
+
// This file's own effective params: its own declared defaults, overridden by whatever the caller passed in.
|
|
50
|
+
const effectiveParams = { ...file.params, ...callParams };
|
|
51
|
+
const beforeSteps = [];
|
|
52
|
+
for (const ref of file.before) {
|
|
53
|
+
const path = await findByName(testsDir, ref.name);
|
|
54
|
+
if (!path) {
|
|
55
|
+
throw new Error(`Before: "${ref.name}" not found in ${testsDir} (referenced by "${file.name}")`);
|
|
56
|
+
}
|
|
57
|
+
const refFile = await loadTestFile(path);
|
|
58
|
+
const resolved = await expand(refFile, ref.params);
|
|
59
|
+
beforeSteps.push(...resolved);
|
|
60
|
+
}
|
|
61
|
+
const ownSteps = file.steps.map((s) => ({ type: s.type, text: substitute(s.text, effectiveParams) }));
|
|
62
|
+
return [...beforeSteps, ...ownSteps];
|
|
63
|
+
}
|
|
64
|
+
return expand(test, {});
|
|
65
|
+
}
|