agentcert 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +84 -0
- package/dist/badge.js +31 -0
- package/dist/bundle.js +80 -0
- package/dist/cli.js +553 -0
- package/dist/corpus-store.js +241 -0
- package/dist/corpus.js +435 -0
- package/dist/failure-review.js +188 -0
- package/dist/index.js +10 -0
- package/dist/lab.js +323 -0
- package/dist/local-server.js +491 -0
- package/dist/monitor.js +100 -0
- package/dist/normalizers.js +193 -0
- package/dist/report.js +148 -0
- package/dist/runner.js +341 -0
- package/dist/schema-validator.js +150 -0
- package/dist/types.js +1 -0
- package/package.json +31 -0
|
@@ -0,0 +1,491 @@
|
|
|
1
|
+
import { createServer } from "node:http";
|
|
2
|
+
import { readFile, stat } from "node:fs/promises";
|
|
3
|
+
import { extname, join, relative, resolve, sep } from "node:path";
|
|
4
|
+
import { pathToFileURL } from "node:url";
|
|
5
|
+
import { reviewedFailureDataset, summarizeCorpus, } from "./corpus.js";
|
|
6
|
+
import { openCorpusStore } from "./corpus-store.js";
|
|
7
|
+
import { applyFailureReviews, appendFailureReview, createFailureReview, findFailurePattern, parseFailureReviewStatus, parseFailureType, parseReviewConfidence, readFailureReviews, } from "./failure-review.js";
|
|
8
|
+
import { buildMonitorSnapshot } from "./monitor.js";
|
|
9
|
+
export async function serveAgentCertMonitor(options) {
|
|
10
|
+
const staticRoot = resolve(options.staticDir);
|
|
11
|
+
const artifactRoot = resolve(options.artifactRoot);
|
|
12
|
+
const server = createServer(async (request, response) => {
|
|
13
|
+
try {
|
|
14
|
+
await handleRequest(request, response, options, staticRoot, artifactRoot);
|
|
15
|
+
}
|
|
16
|
+
catch (error) {
|
|
17
|
+
sendJson(response, 500, { error: error instanceof Error ? error.message : String(error) });
|
|
18
|
+
}
|
|
19
|
+
});
|
|
20
|
+
await new Promise((resolveListen) => server.listen(options.port, options.host, resolveListen));
|
|
21
|
+
const address = `http://${options.host}:${options.port}`;
|
|
22
|
+
process.stdout.write(`AgentCert Monitor server listening at ${address}\n`);
|
|
23
|
+
process.stdout.write(`Serving dashboard from ${staticRoot}\n`);
|
|
24
|
+
process.stdout.write(`Serving artifacts from ${artifactRoot}\n`);
|
|
25
|
+
}
|
|
26
|
+
async function handleRequest(request, response, options, staticRoot, artifactRoot) {
|
|
27
|
+
if (!request.url) {
|
|
28
|
+
sendJson(response, 400, { error: "Missing URL." });
|
|
29
|
+
return;
|
|
30
|
+
}
|
|
31
|
+
const url = new URL(request.url, `http://${request.headers.host ?? `${options.host}:${options.port}`}`);
|
|
32
|
+
if (url.pathname === "/api/health") {
|
|
33
|
+
await withStore(options.store, async (store) => {
|
|
34
|
+
sendJson(response, 200, {
|
|
35
|
+
ok: true,
|
|
36
|
+
subject: options.subject,
|
|
37
|
+
store: { kind: store.kind, description: store.description },
|
|
38
|
+
artifactRoot,
|
|
39
|
+
});
|
|
40
|
+
});
|
|
41
|
+
return;
|
|
42
|
+
}
|
|
43
|
+
if (url.pathname === "/api/monitor") {
|
|
44
|
+
await withStore(options.store, async (store) => {
|
|
45
|
+
const records = applyFailureReviews(await store.readAll(), await readFailureReviews(options.reviewsPath));
|
|
46
|
+
sendJson(response, 200, buildMonitorSnapshot(records, { subject: options.subject, detailUrl: options.detailUrl }));
|
|
47
|
+
});
|
|
48
|
+
return;
|
|
49
|
+
}
|
|
50
|
+
if (url.pathname === "/api/runs") {
|
|
51
|
+
await withStore(options.store, async (store) => {
|
|
52
|
+
const records = applyFailureReviews(await store.readAll(), await readFailureReviews(options.reviewsPath));
|
|
53
|
+
sendJson(response, 200, { runs: records.sort((left, right) => right.timestamp.localeCompare(left.timestamp)) });
|
|
54
|
+
});
|
|
55
|
+
return;
|
|
56
|
+
}
|
|
57
|
+
if (url.pathname === "/api/corpus/metrics") {
|
|
58
|
+
await withStore(options.store, async (store) => {
|
|
59
|
+
const records = applyFailureReviews(await store.readAll(), await readFailureReviews(options.reviewsPath));
|
|
60
|
+
sendJson(response, 200, summarizeCorpus(records).taxonomy);
|
|
61
|
+
});
|
|
62
|
+
return;
|
|
63
|
+
}
|
|
64
|
+
if (url.pathname === "/api/corpus/reviewed-dataset") {
|
|
65
|
+
await withStore(options.store, async (store) => {
|
|
66
|
+
const records = applyFailureReviews(await store.readAll(), await readFailureReviews(options.reviewsPath));
|
|
67
|
+
const rows = reviewedFailureDataset(records);
|
|
68
|
+
response.writeHead(200, {
|
|
69
|
+
"content-type": "application/x-ndjson; charset=utf-8",
|
|
70
|
+
"cache-control": "no-cache",
|
|
71
|
+
"content-disposition": 'attachment; filename="agentcert-reviewed-failure-dataset.jsonl"',
|
|
72
|
+
});
|
|
73
|
+
const payload = rows.map((row) => JSON.stringify(row)).join("\n");
|
|
74
|
+
response.end(`${payload}${payload.length > 0 ? "\n" : ""}`);
|
|
75
|
+
});
|
|
76
|
+
return;
|
|
77
|
+
}
|
|
78
|
+
const reviewRoute = matchRunReviewRoute(url.pathname);
|
|
79
|
+
if (reviewRoute && request.method === "POST") {
|
|
80
|
+
await handleFailureReviewPost(request, response, options, artifactRoot, reviewRoute.runId);
|
|
81
|
+
return;
|
|
82
|
+
}
|
|
83
|
+
if (url.pathname.startsWith("/api/runs/") && request.method !== "POST") {
|
|
84
|
+
const runId = decodeURIComponent(url.pathname.slice("/api/runs/".length));
|
|
85
|
+
await withStore(options.store, async (store) => {
|
|
86
|
+
const records = applyFailureReviews(await store.readAll(), await readFailureReviews(options.reviewsPath));
|
|
87
|
+
const record = records.find((item) => item.id === runId || item.runId === runId);
|
|
88
|
+
if (!record) {
|
|
89
|
+
sendJson(response, 404, { error: `Run ${runId} was not found.` });
|
|
90
|
+
return;
|
|
91
|
+
}
|
|
92
|
+
sendJson(response, 200, await buildRunDetail(record, artifactRoot));
|
|
93
|
+
});
|
|
94
|
+
return;
|
|
95
|
+
}
|
|
96
|
+
if (url.pathname === "/api/artifacts") {
|
|
97
|
+
await serveArtifact(response, artifactRoot, url.searchParams.get("path"));
|
|
98
|
+
return;
|
|
99
|
+
}
|
|
100
|
+
await serveStatic(response, staticRoot, url.pathname);
|
|
101
|
+
}
|
|
102
|
+
async function handleFailureReviewPost(request, response, options, artifactRoot, runId) {
|
|
103
|
+
const reviewsPath = options.reviewsPath;
|
|
104
|
+
if (!reviewsPath) {
|
|
105
|
+
sendJson(response, 400, { error: "This server was not configured with a failure review ledger path." });
|
|
106
|
+
return;
|
|
107
|
+
}
|
|
108
|
+
const body = await readJsonBody(request);
|
|
109
|
+
const patternKey = stringField(body, "patternKey");
|
|
110
|
+
const type = parseFailureType(stringField(body, "type"));
|
|
111
|
+
const reviewer = stringField(body, "reviewer") ?? "local-reviewer";
|
|
112
|
+
const note = stringField(body, "note");
|
|
113
|
+
const explicitStatus = stringField(body, "status");
|
|
114
|
+
const confidence = parseReviewConfidence(numberField(body, "confidence"));
|
|
115
|
+
const evidenceContext = reviewEvidenceContextField(body);
|
|
116
|
+
const taxonomyRationale = reviewTaxonomyRationaleField(body);
|
|
117
|
+
if (!patternKey) {
|
|
118
|
+
sendJson(response, 400, { error: "Missing patternKey." });
|
|
119
|
+
return;
|
|
120
|
+
}
|
|
121
|
+
await withStore(options.store, async (store) => {
|
|
122
|
+
const records = await store.readAll();
|
|
123
|
+
const record = records.find((item) => item.id === runId || item.runId === runId);
|
|
124
|
+
if (!record) {
|
|
125
|
+
sendJson(response, 404, { error: `Run ${runId} was not found.` });
|
|
126
|
+
return;
|
|
127
|
+
}
|
|
128
|
+
const target = {
|
|
129
|
+
patternKey,
|
|
130
|
+
recordId: record.id,
|
|
131
|
+
runId: record.runId,
|
|
132
|
+
product: record.product,
|
|
133
|
+
scenarioName: record.scenarioName,
|
|
134
|
+
faultName: record.faultName,
|
|
135
|
+
};
|
|
136
|
+
const matched = findFailurePattern(records, target);
|
|
137
|
+
if (!matched) {
|
|
138
|
+
sendJson(response, 404, { error: `Failure pattern ${patternKey} was not found on run ${runId}.` });
|
|
139
|
+
return;
|
|
140
|
+
}
|
|
141
|
+
const suggestedType = matched.pattern.suggestedType ?? matched.pattern.type;
|
|
142
|
+
const review = createFailureReview({
|
|
143
|
+
target,
|
|
144
|
+
type,
|
|
145
|
+
status: parseFailureReviewStatus(explicitStatus, type, suggestedType),
|
|
146
|
+
suggestedType,
|
|
147
|
+
reviewer,
|
|
148
|
+
note,
|
|
149
|
+
confidence,
|
|
150
|
+
evidenceContext,
|
|
151
|
+
taxonomyRationale,
|
|
152
|
+
});
|
|
153
|
+
await appendFailureReview(reviewsPath, review);
|
|
154
|
+
const updated = applyFailureReviews(records, await readFailureReviews(reviewsPath));
|
|
155
|
+
await store.append(updated, { replace: true });
|
|
156
|
+
const updatedRecord = updated.find((item) => item.id === record.id) ?? record;
|
|
157
|
+
sendJson(response, 200, await buildRunDetail(updatedRecord, artifactRoot));
|
|
158
|
+
});
|
|
159
|
+
}
|
|
160
|
+
export async function buildRunDetail(record, artifactRoot) {
|
|
161
|
+
const root = resolve(artifactRoot);
|
|
162
|
+
const tracePath = normalizeArtifactPath(record.artifacts.trace);
|
|
163
|
+
const resultPath = normalizeArtifactPath(record.artifacts.result);
|
|
164
|
+
const trace = tracePath ? await readJsonIfExists(join(root, tracePath)) : undefined;
|
|
165
|
+
const result = await readTripwireResult(root, resultPath);
|
|
166
|
+
const tripwireRun = result?.runs?.find((run) => run.runId === record.runId);
|
|
167
|
+
const assertions = tripwireRun?.assertions ?? assertionsFromFailurePatterns(record.failurePatterns);
|
|
168
|
+
const artifacts = await collectArtifacts(record, root, trace);
|
|
169
|
+
return {
|
|
170
|
+
record,
|
|
171
|
+
failurePatterns: record.failurePatterns,
|
|
172
|
+
assertions,
|
|
173
|
+
timeline: buildTimeline(record, assertions, trace),
|
|
174
|
+
artifacts,
|
|
175
|
+
traceSummary: traceSummary(trace),
|
|
176
|
+
finalUrl: tripwireRun?.finalUrl ?? stringMetadata(record.metadata, "finalUrl"),
|
|
177
|
+
diagnostics: stringArray(tripwireRun?.diagnostics ?? record.metadata?.diagnostics),
|
|
178
|
+
warnings: stringArray(tripwireRun?.warnings ?? record.metadata?.warnings ?? trace?.warnings),
|
|
179
|
+
};
|
|
180
|
+
}
|
|
181
|
+
function buildTimeline(record, assertions, trace) {
|
|
182
|
+
const failedAssertions = assertions.filter((assertion) => assertion.pass === false);
|
|
183
|
+
const timeline = failedAssertions.map((assertion) => ({
|
|
184
|
+
kind: "failure",
|
|
185
|
+
title: "Assertion failure",
|
|
186
|
+
timestamp: record.timestamp,
|
|
187
|
+
message: `${assertion.message}${assertion.observed === undefined ? "" : ` Observed: ${stringifyShort(assertion.observed)}`}`,
|
|
188
|
+
}));
|
|
189
|
+
const steps = trace?.steps ?? [];
|
|
190
|
+
const firstChangedStep = firstDivergenceStep(steps);
|
|
191
|
+
if (firstChangedStep) {
|
|
192
|
+
timeline.push({
|
|
193
|
+
kind: "page-state",
|
|
194
|
+
title: "First observed page divergence",
|
|
195
|
+
timestamp: firstChangedStep.timestamp,
|
|
196
|
+
stepIndex: firstChangedStep.stepIndex,
|
|
197
|
+
message: firstChangedStep.visibleTextSample ?? "The page state changed before the task reached the expected outcome.",
|
|
198
|
+
artifactPath: firstChangedStep.screenshotPath,
|
|
199
|
+
});
|
|
200
|
+
}
|
|
201
|
+
for (const event of steps.flatMap((step) => (step.agentEvents ?? []).map((agentEvent) => ({
|
|
202
|
+
...agentEvent,
|
|
203
|
+
stepIndex: step.stepIndex,
|
|
204
|
+
})))) {
|
|
205
|
+
timeline.push({
|
|
206
|
+
kind: "agent-action",
|
|
207
|
+
title: `Agent ${event.type ?? "event"}`,
|
|
208
|
+
timestamp: event.timestamp,
|
|
209
|
+
stepIndex: event.stepIndex,
|
|
210
|
+
message: [event.target, event.note].filter(Boolean).join(" - "),
|
|
211
|
+
});
|
|
212
|
+
}
|
|
213
|
+
for (const [index, item] of [...(trace?.networkErrors ?? []), ...steps.flatMap((step) => step.networkErrors ?? [])].entries()) {
|
|
214
|
+
timeline.push({ kind: "network", title: "Network error", message: stringifyShort(item), stepIndex: index + 1 });
|
|
215
|
+
}
|
|
216
|
+
for (const [index, item] of [...(trace?.consoleErrors ?? []), ...steps.flatMap((step) => step.consoleErrors ?? [])].entries()) {
|
|
217
|
+
timeline.push({ kind: "console", title: "Console error", message: stringifyShort(item), stepIndex: index + 1 });
|
|
218
|
+
}
|
|
219
|
+
return timeline.sort((left, right) => (left.timestamp ?? "").localeCompare(right.timestamp ?? ""));
|
|
220
|
+
}
|
|
221
|
+
async function collectArtifacts(record, root, trace) {
|
|
222
|
+
const artifactPaths = new Map();
|
|
223
|
+
const tracePath = normalizeArtifactPath(record.artifacts.trace);
|
|
224
|
+
const resultPath = normalizeArtifactPath(record.artifacts.result);
|
|
225
|
+
const artifactDir = normalizeArtifactPath(record.artifacts.artifactDir);
|
|
226
|
+
if (tracePath)
|
|
227
|
+
artifactPaths.set(tracePath, "trace");
|
|
228
|
+
if (resultPath)
|
|
229
|
+
artifactPaths.set(resultPath, "json");
|
|
230
|
+
if (artifactDir)
|
|
231
|
+
artifactPaths.set(join(artifactDir, "agent-events.jsonl"), "events");
|
|
232
|
+
for (const step of trace?.steps ?? []) {
|
|
233
|
+
if (step.screenshotPath && tracePath)
|
|
234
|
+
artifactPaths.set(join(dirnameLike(tracePath), step.screenshotPath), "screenshot");
|
|
235
|
+
if (step.domSnapshotPath && tracePath)
|
|
236
|
+
artifactPaths.set(join(dirnameLike(tracePath), step.domSnapshotPath), "dom");
|
|
237
|
+
}
|
|
238
|
+
const artifacts = [];
|
|
239
|
+
for (const [path, kind] of artifactPaths) {
|
|
240
|
+
const normalized = normalizeArtifactPath(path);
|
|
241
|
+
if (!normalized)
|
|
242
|
+
continue;
|
|
243
|
+
const fullPath = resolveInside(root, normalized);
|
|
244
|
+
if (!fullPath)
|
|
245
|
+
continue;
|
|
246
|
+
const fileStat = await stat(fullPath).catch(() => undefined);
|
|
247
|
+
if (!fileStat?.isFile())
|
|
248
|
+
continue;
|
|
249
|
+
artifacts.push({
|
|
250
|
+
label: normalized.split(/[\\/]/).slice(-2).join("/"),
|
|
251
|
+
kind,
|
|
252
|
+
path: normalized,
|
|
253
|
+
url: `/api/artifacts?path=${encodeURIComponent(normalized)}`,
|
|
254
|
+
sizeBytes: fileStat.size,
|
|
255
|
+
});
|
|
256
|
+
}
|
|
257
|
+
return artifacts.sort((left, right) => artifactKindRank(left.kind) - artifactKindRank(right.kind) || left.path.localeCompare(right.path));
|
|
258
|
+
}
|
|
259
|
+
async function serveArtifact(response, artifactRoot, path) {
|
|
260
|
+
if (!path) {
|
|
261
|
+
sendJson(response, 400, { error: "Missing artifact path." });
|
|
262
|
+
return;
|
|
263
|
+
}
|
|
264
|
+
const normalized = normalizeArtifactPath(path);
|
|
265
|
+
const fullPath = normalized ? resolveInside(artifactRoot, normalized) : undefined;
|
|
266
|
+
if (!fullPath) {
|
|
267
|
+
sendJson(response, 403, { error: "Artifact path is outside the artifact root." });
|
|
268
|
+
return;
|
|
269
|
+
}
|
|
270
|
+
try {
|
|
271
|
+
const file = await readFile(fullPath);
|
|
272
|
+
response.writeHead(200, {
|
|
273
|
+
"content-type": mimeType(fullPath),
|
|
274
|
+
"cache-control": "no-cache",
|
|
275
|
+
});
|
|
276
|
+
response.end(file);
|
|
277
|
+
}
|
|
278
|
+
catch {
|
|
279
|
+
sendJson(response, 404, { error: "Artifact was not found." });
|
|
280
|
+
}
|
|
281
|
+
}
|
|
282
|
+
async function serveStatic(response, staticRoot, pathname) {
|
|
283
|
+
const requested = pathname === "/" ? "index.html" : pathname.replace(/^\/+/, "");
|
|
284
|
+
const fullPath = resolveInside(staticRoot, requested);
|
|
285
|
+
if (!fullPath) {
|
|
286
|
+
sendJson(response, 403, { error: "Static path is outside the dashboard root." });
|
|
287
|
+
return;
|
|
288
|
+
}
|
|
289
|
+
try {
|
|
290
|
+
const fileStat = await stat(fullPath);
|
|
291
|
+
if (fileStat.isDirectory()) {
|
|
292
|
+
await sendFile(response, join(fullPath, "index.html"));
|
|
293
|
+
return;
|
|
294
|
+
}
|
|
295
|
+
await sendFile(response, fullPath);
|
|
296
|
+
}
|
|
297
|
+
catch {
|
|
298
|
+
await sendFile(response, join(staticRoot, "index.html"));
|
|
299
|
+
}
|
|
300
|
+
}
|
|
301
|
+
async function sendFile(response, path) {
|
|
302
|
+
const file = await readFile(path);
|
|
303
|
+
response.writeHead(200, { "content-type": mimeType(path), "cache-control": "no-cache" });
|
|
304
|
+
response.end(file);
|
|
305
|
+
}
|
|
306
|
+
async function withStore(options, callback) {
|
|
307
|
+
const store = await openCorpusStore(options);
|
|
308
|
+
try {
|
|
309
|
+
return await callback(store);
|
|
310
|
+
}
|
|
311
|
+
finally {
|
|
312
|
+
await store.close();
|
|
313
|
+
}
|
|
314
|
+
}
|
|
315
|
+
async function readTripwireResult(root, resultPath) {
|
|
316
|
+
const candidates = [resultPath, "tripwire-result.json"].filter(Boolean);
|
|
317
|
+
for (const candidate of candidates) {
|
|
318
|
+
const normalized = normalizeArtifactPath(candidate);
|
|
319
|
+
const fullPath = normalized ? resolveInside(root, normalized) : undefined;
|
|
320
|
+
const result = fullPath ? await readJsonIfExists(fullPath) : undefined;
|
|
321
|
+
if (result)
|
|
322
|
+
return result;
|
|
323
|
+
}
|
|
324
|
+
return undefined;
|
|
325
|
+
}
|
|
326
|
+
async function readJsonIfExists(path) {
|
|
327
|
+
try {
|
|
328
|
+
return JSON.parse(await readFile(path, "utf8"));
|
|
329
|
+
}
|
|
330
|
+
catch {
|
|
331
|
+
return undefined;
|
|
332
|
+
}
|
|
333
|
+
}
|
|
334
|
+
function assertionsFromFailurePatterns(patterns) {
|
|
335
|
+
return patterns.map((pattern) => ({ type: pattern.key, pass: false, message: pattern.message }));
|
|
336
|
+
}
|
|
337
|
+
function traceSummary(trace) {
|
|
338
|
+
const steps = trace?.steps ?? [];
|
|
339
|
+
if (steps.length === 0)
|
|
340
|
+
return undefined;
|
|
341
|
+
return {
|
|
342
|
+
stepCount: steps.length,
|
|
343
|
+
firstStepText: steps[0]?.visibleTextSample,
|
|
344
|
+
lastStepText: steps[steps.length - 1]?.visibleTextSample,
|
|
345
|
+
firstDivergenceStep: firstDivergenceStep(steps)?.stepIndex,
|
|
346
|
+
};
|
|
347
|
+
}
|
|
348
|
+
function firstDivergenceStep(steps) {
|
|
349
|
+
const firstHash = steps[0]?.visibleTextSample;
|
|
350
|
+
return steps.find((step) => step.visibleTextSample && step.visibleTextSample !== firstHash);
|
|
351
|
+
}
|
|
352
|
+
function normalizeArtifactPath(input) {
|
|
353
|
+
if (!input)
|
|
354
|
+
return undefined;
|
|
355
|
+
const normalized = input.replace(/\\/g, "/");
|
|
356
|
+
if (normalized.includes(".."))
|
|
357
|
+
return undefined;
|
|
358
|
+
if (normalized.startsWith("packages/tripwire-ci/.tripwire/public-demo/")) {
|
|
359
|
+
return normalized.slice("packages/tripwire-ci/.tripwire/public-demo/".length);
|
|
360
|
+
}
|
|
361
|
+
return normalized.replace(/^\/+/, "");
|
|
362
|
+
}
|
|
363
|
+
function resolveInside(root, path) {
|
|
364
|
+
const fullPath = resolve(root, path);
|
|
365
|
+
const rel = relative(root, fullPath);
|
|
366
|
+
if (rel.startsWith("..") || rel === ".." || rel.split(sep).includes("..")) {
|
|
367
|
+
return undefined;
|
|
368
|
+
}
|
|
369
|
+
return fullPath;
|
|
370
|
+
}
|
|
371
|
+
function dirnameLike(path) {
|
|
372
|
+
const parts = path.split(/[\\/]/);
|
|
373
|
+
parts.pop();
|
|
374
|
+
return parts.join("/");
|
|
375
|
+
}
|
|
376
|
+
function artifactKindRank(kind) {
|
|
377
|
+
return ["screenshot", "dom", "trace", "json", "events", "report", "other"].indexOf(kind);
|
|
378
|
+
}
|
|
379
|
+
function sendJson(response, status, data) {
|
|
380
|
+
response.writeHead(status, { "content-type": "application/json; charset=utf-8", "cache-control": "no-cache" });
|
|
381
|
+
response.end(`${JSON.stringify(data, null, 2)}\n`);
|
|
382
|
+
}
|
|
383
|
+
async function readJsonBody(request) {
|
|
384
|
+
const chunks = [];
|
|
385
|
+
let totalBytes = 0;
|
|
386
|
+
for await (const chunk of request) {
|
|
387
|
+
const buffer = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk);
|
|
388
|
+
totalBytes += buffer.length;
|
|
389
|
+
if (totalBytes > 64 * 1024) {
|
|
390
|
+
throw new Error("Request body is too large.");
|
|
391
|
+
}
|
|
392
|
+
chunks.push(buffer);
|
|
393
|
+
}
|
|
394
|
+
if (chunks.length === 0)
|
|
395
|
+
return {};
|
|
396
|
+
const raw = Buffer.concat(chunks).toString("utf8");
|
|
397
|
+
return JSON.parse(raw);
|
|
398
|
+
}
|
|
399
|
+
function stringField(input, key) {
|
|
400
|
+
const value = input[key];
|
|
401
|
+
return typeof value === "string" && value.length > 0 ? value : undefined;
|
|
402
|
+
}
|
|
403
|
+
function numberField(input, key) {
|
|
404
|
+
const value = input[key];
|
|
405
|
+
return typeof value === "number" && Number.isFinite(value) ? value : undefined;
|
|
406
|
+
}
|
|
407
|
+
function recordField(input, key) {
|
|
408
|
+
const value = input[key];
|
|
409
|
+
return value && typeof value === "object" && !Array.isArray(value) ? value : {};
|
|
410
|
+
}
|
|
411
|
+
function stringArrayField(input, key) {
|
|
412
|
+
const value = input[key];
|
|
413
|
+
return Array.isArray(value) ? value.filter((item) => typeof item === "string" && item.length > 0) : [];
|
|
414
|
+
}
|
|
415
|
+
function reviewEvidenceContextField(input) {
|
|
416
|
+
const nested = recordField(input, "evidenceContext");
|
|
417
|
+
const firstDivergenceSnippet = stringField(nested, "firstDivergenceSnippet") ?? stringField(input, "firstDivergenceSnippet");
|
|
418
|
+
const screenshotPath = stringField(nested, "screenshotPath") ?? stringField(input, "screenshotPath");
|
|
419
|
+
const screenshotUrl = stringField(nested, "screenshotUrl") ?? stringField(input, "screenshotUrl");
|
|
420
|
+
const tracePath = stringField(nested, "tracePath") ?? stringField(input, "tracePath");
|
|
421
|
+
const stepIndex = numberField(nested, "stepIndex") ?? numberField(input, "stepIndex");
|
|
422
|
+
if (!firstDivergenceSnippet && !screenshotPath && !screenshotUrl && !tracePath && stepIndex === undefined) {
|
|
423
|
+
return undefined;
|
|
424
|
+
}
|
|
425
|
+
return {
|
|
426
|
+
firstDivergenceSnippet,
|
|
427
|
+
screenshotPath,
|
|
428
|
+
screenshotUrl,
|
|
429
|
+
tracePath,
|
|
430
|
+
stepIndex,
|
|
431
|
+
};
|
|
432
|
+
}
|
|
433
|
+
function reviewTaxonomyRationaleField(input) {
|
|
434
|
+
const nested = recordField(input, "taxonomyRationale");
|
|
435
|
+
const primaryReason = stringField(nested, "primaryReason") ?? stringField(input, "why");
|
|
436
|
+
if (!primaryReason) {
|
|
437
|
+
return undefined;
|
|
438
|
+
}
|
|
439
|
+
const supportingSignals = stringArrayField(nested, "supportingSignals");
|
|
440
|
+
const contradictingSignals = stringArrayField(nested, "contradictingSignals");
|
|
441
|
+
return {
|
|
442
|
+
primaryReason,
|
|
443
|
+
supportingSignals: supportingSignals.length > 0 ? supportingSignals : undefined,
|
|
444
|
+
contradictingSignals: contradictingSignals.length > 0 ? contradictingSignals : undefined,
|
|
445
|
+
classifierLimitation: stringField(nested, "classifierLimitation"),
|
|
446
|
+
};
|
|
447
|
+
}
|
|
448
|
+
function matchRunReviewRoute(pathname) {
|
|
449
|
+
const suffix = "/failure-reviews";
|
|
450
|
+
if (!pathname.startsWith("/api/runs/") || !pathname.endsWith(suffix)) {
|
|
451
|
+
return undefined;
|
|
452
|
+
}
|
|
453
|
+
const encoded = pathname.slice("/api/runs/".length, -suffix.length);
|
|
454
|
+
return { runId: decodeURIComponent(encoded) };
|
|
455
|
+
}
|
|
456
|
+
function stringMetadata(metadata, key) {
|
|
457
|
+
const value = metadata?.[key];
|
|
458
|
+
return typeof value === "string" ? value : undefined;
|
|
459
|
+
}
|
|
460
|
+
function stringArray(input) {
|
|
461
|
+
return Array.isArray(input) ? input.filter((item) => typeof item === "string") : [];
|
|
462
|
+
}
|
|
463
|
+
function stringifyShort(input) {
|
|
464
|
+
const value = typeof input === "string" ? input : JSON.stringify(input);
|
|
465
|
+
return value.length > 500 ? `${value.slice(0, 500)}...` : value;
|
|
466
|
+
}
|
|
467
|
+
function mimeType(path) {
|
|
468
|
+
switch (extname(path).toLowerCase()) {
|
|
469
|
+
case ".css":
|
|
470
|
+
return "text/css; charset=utf-8";
|
|
471
|
+
case ".html":
|
|
472
|
+
return "text/html; charset=utf-8";
|
|
473
|
+
case ".js":
|
|
474
|
+
return "text/javascript; charset=utf-8";
|
|
475
|
+
case ".json":
|
|
476
|
+
return "application/json; charset=utf-8";
|
|
477
|
+
case ".jsonl":
|
|
478
|
+
return "application/x-ndjson; charset=utf-8";
|
|
479
|
+
case ".png":
|
|
480
|
+
return "image/png";
|
|
481
|
+
case ".svg":
|
|
482
|
+
return "image/svg+xml";
|
|
483
|
+
case ".txt":
|
|
484
|
+
return "text/plain; charset=utf-8";
|
|
485
|
+
default:
|
|
486
|
+
return "application/octet-stream";
|
|
487
|
+
}
|
|
488
|
+
}
|
|
489
|
+
export function dashboardFileUrl(staticDir) {
|
|
490
|
+
return pathToFileURL(resolve(staticDir, "index.html")).toString();
|
|
491
|
+
}
|
package/dist/monitor.js
ADDED
|
@@ -0,0 +1,100 @@
|
|
|
1
|
+
import { mkdir, writeFile } from "node:fs/promises";
|
|
2
|
+
import { dirname, resolve } from "node:path";
|
|
3
|
+
import { summarizeCorpus } from "./corpus.js";
|
|
4
|
+
export function buildMonitorSnapshot(records, options) {
|
|
5
|
+
const summary = summarizeCorpus(records);
|
|
6
|
+
return {
|
|
7
|
+
schemaVersion: "1",
|
|
8
|
+
kind: "agentcert.monitor_snapshot",
|
|
9
|
+
generatedAt: new Date().toISOString(),
|
|
10
|
+
subject: options.subject,
|
|
11
|
+
summary,
|
|
12
|
+
filters: buildFilters(records),
|
|
13
|
+
lifecycle: [
|
|
14
|
+
lifecycleGate(records, {
|
|
15
|
+
id: "mcpbench",
|
|
16
|
+
name: "MCPBench",
|
|
17
|
+
phase: "Before release",
|
|
18
|
+
description: "MCP servers, exposed tools, policy behavior, and runtime traces.",
|
|
19
|
+
}),
|
|
20
|
+
lifecycleGate(records, {
|
|
21
|
+
id: "tripwire-ci",
|
|
22
|
+
name: "Tripwire CI",
|
|
23
|
+
phase: "Before release",
|
|
24
|
+
description: "Browser and computer-use agents under adversarial UI and network faults.",
|
|
25
|
+
}),
|
|
26
|
+
lifecycleGate(records, {
|
|
27
|
+
id: "onegent-runtime",
|
|
28
|
+
name: "Onegent Runtime",
|
|
29
|
+
phase: "After release",
|
|
30
|
+
description: "High-risk live actions requiring approval, verification, and audit.",
|
|
31
|
+
}),
|
|
32
|
+
],
|
|
33
|
+
recentRuns: [...records]
|
|
34
|
+
.sort((left, right) => right.timestamp.localeCompare(left.timestamp))
|
|
35
|
+
.slice(0, 50)
|
|
36
|
+
.map((record) => ({
|
|
37
|
+
id: record.id,
|
|
38
|
+
product: record.product,
|
|
39
|
+
phase: record.phase,
|
|
40
|
+
subject: record.subject,
|
|
41
|
+
agentName: record.agentName,
|
|
42
|
+
agentVersion: record.agentVersion,
|
|
43
|
+
scenarioName: record.scenarioName,
|
|
44
|
+
faultName: record.faultName,
|
|
45
|
+
failureTypes: [...new Set(record.failurePatterns.map((pattern) => pattern.type))],
|
|
46
|
+
passed: record.passed,
|
|
47
|
+
score: record.score,
|
|
48
|
+
timestamp: record.timestamp,
|
|
49
|
+
durationMs: record.durationMs,
|
|
50
|
+
evidenceCount: record.evidenceCount,
|
|
51
|
+
primaryFailure: record.failurePatterns[0]?.message,
|
|
52
|
+
failurePatterns: record.failurePatterns,
|
|
53
|
+
taxonomyReviewStatus: taxonomyReviewStatus(record.failurePatterns),
|
|
54
|
+
reviewedFailureCount: record.failurePatterns.filter((pattern) => pattern.reviewStatus === "confirmed" || pattern.reviewStatus === "corrected").length,
|
|
55
|
+
unreviewedFailureCount: record.failurePatterns.filter((pattern) => (pattern.reviewStatus ?? "unreviewed") === "unreviewed").length,
|
|
56
|
+
artifacts: record.artifacts,
|
|
57
|
+
})),
|
|
58
|
+
failurePatterns: summary.topFailurePatterns,
|
|
59
|
+
links: {
|
|
60
|
+
detailUrl: options.detailUrl,
|
|
61
|
+
},
|
|
62
|
+
};
|
|
63
|
+
}
|
|
64
|
+
function buildFilters(records) {
|
|
65
|
+
return {
|
|
66
|
+
agents: unique(records.map((record) => record.agentName)),
|
|
67
|
+
faults: unique(records.map((record) => record.faultName).filter((value) => Boolean(value))),
|
|
68
|
+
versions: unique(records.map((record) => record.agentVersion)),
|
|
69
|
+
failureTypes: unique(records.flatMap((record) => record.failurePatterns.map((pattern) => pattern.type))),
|
|
70
|
+
products: unique(records.map((record) => record.product)),
|
|
71
|
+
reviewStatuses: unique(records.map((record) => taxonomyReviewStatus(record.failurePatterns))),
|
|
72
|
+
};
|
|
73
|
+
}
|
|
74
|
+
function taxonomyReviewStatus(patterns) {
|
|
75
|
+
if (patterns.length === 0)
|
|
76
|
+
return "none";
|
|
77
|
+
return patterns.every((pattern) => pattern.reviewStatus === "confirmed" || pattern.reviewStatus === "corrected") ? "reviewed" : "needs_review";
|
|
78
|
+
}
|
|
79
|
+
function unique(values) {
|
|
80
|
+
return [...new Set(values)].sort((left, right) => left.localeCompare(right));
|
|
81
|
+
}
|
|
82
|
+
export async function writeMonitorSnapshot(path, snapshot) {
|
|
83
|
+
const outPath = resolve(path);
|
|
84
|
+
await mkdir(dirname(outPath), { recursive: true });
|
|
85
|
+
await writeFile(outPath, `${JSON.stringify(snapshot, null, 2)}\n`);
|
|
86
|
+
}
|
|
87
|
+
function lifecycleGate(records, gate) {
|
|
88
|
+
const gateRecords = records.filter((record) => record.product === gate.id);
|
|
89
|
+
const passedCount = gateRecords.filter((record) => record.passed).length;
|
|
90
|
+
const failedCount = gateRecords.length - passedCount;
|
|
91
|
+
const passRate = gateRecords.length === 0 ? 0 : passedCount / gateRecords.length;
|
|
92
|
+
return {
|
|
93
|
+
...gate,
|
|
94
|
+
recordCount: gateRecords.length,
|
|
95
|
+
passedCount,
|
|
96
|
+
failedCount,
|
|
97
|
+
passRate,
|
|
98
|
+
status: gateRecords.length === 0 ? "waiting" : passRate >= 0.8 ? "passing" : "failing",
|
|
99
|
+
};
|
|
100
|
+
}
|