@kirrosh/zond 0.26.0 → 0.27.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +59 -0
- package/README.md +22 -21
- package/bin/zond.mjs +23 -0
- package/package.json +13 -16
- package/scripts/npm/postinstall.mjs +76 -0
- package/src/CLAUDE.md +0 -112
- package/src/bun-types.d.ts +0 -5
- package/src/cli/argv.ts +0 -122
- package/src/cli/commands/add-api.ts +0 -146
- package/src/cli/commands/api/annotate/idempotency.ts +0 -59
- package/src/cli/commands/api/annotate/index.ts +0 -880
- package/src/cli/commands/api/annotate/lifecycle.ts +0 -74
- package/src/cli/commands/api/annotate/overlay.ts +0 -206
- package/src/cli/commands/api/annotate/pagination.ts +0 -64
- package/src/cli/commands/api/annotate/prompts.ts +0 -220
- package/src/cli/commands/api/annotate/readback.ts +0 -58
- package/src/cli/commands/api/annotate/resources.ts +0 -91
- package/src/cli/commands/api/annotate/seed-bodies.ts +0 -61
- package/src/cli/commands/audit.ts +0 -786
- package/src/cli/commands/catalog.ts +0 -97
- package/src/cli/commands/check.ts +0 -361
- package/src/cli/commands/checks.ts +0 -1072
- package/src/cli/commands/ci-init.ts +0 -223
- package/src/cli/commands/clean.ts +0 -212
- package/src/cli/commands/cleanup.ts +0 -236
- package/src/cli/commands/completions.ts +0 -192
- package/src/cli/commands/coverage.ts +0 -872
- package/src/cli/commands/db.ts +0 -611
- package/src/cli/commands/describe.ts +0 -120
- package/src/cli/commands/discover.ts +0 -1356
- package/src/cli/commands/doctor.ts +0 -661
- package/src/cli/commands/fixtures.ts +0 -402
- package/src/cli/commands/generate.ts +0 -577
- package/src/cli/commands/init/agents-md.ts +0 -61
- package/src/cli/commands/init/bootstrap.ts +0 -111
- package/src/cli/commands/init/index.ts +0 -244
- package/src/cli/commands/init/skills.ts +0 -141
- package/src/cli/commands/init/templates/agents.md +0 -86
- package/src/cli/commands/init/templates/markdown.d.ts +0 -4
- package/src/cli/commands/init/templates/skills/warm-up-target.md +0 -122
- package/src/cli/commands/init/templates/skills/zond-checks.md +0 -621
- package/src/cli/commands/init/templates/skills/zond-seed.md +0 -114
- package/src/cli/commands/init/templates/skills/zond-triage.md +0 -272
- package/src/cli/commands/init/templates/skills/zond.md +0 -861
- package/src/cli/commands/init/templates/zond-config.yml +0 -14
- package/src/cli/commands/prepare-fixtures.ts +0 -97
- package/src/cli/commands/probe/_seed-bodies.ts +0 -52
- package/src/cli/commands/probe/mass-assignment.ts +0 -594
- package/src/cli/commands/probe/security.ts +0 -537
- package/src/cli/commands/probe/static.ts +0 -255
- package/src/cli/commands/probe/webhooks.ts +0 -163
- package/src/cli/commands/probe.ts +0 -535
- package/src/cli/commands/reference.ts +0 -87
- package/src/cli/commands/refresh-api.ts +0 -227
- package/src/cli/commands/remove-api.ts +0 -150
- package/src/cli/commands/report-bundle.ts +0 -310
- package/src/cli/commands/report.ts +0 -241
- package/src/cli/commands/request.ts +0 -548
- package/src/cli/commands/run.ts +0 -1142
- package/src/cli/commands/schema-from-runs.ts +0 -128
- package/src/cli/commands/secrets.ts +0 -133
- package/src/cli/commands/session.ts +0 -244
- package/src/cli/commands/use.ts +0 -74
- package/src/cli/index.ts +0 -55
- package/src/cli/json-envelope.ts +0 -108
- package/src/cli/json-schemas.ts +0 -314
- package/src/cli/output.ts +0 -40
- package/src/cli/program.ts +0 -219
- package/src/cli/resolve.ts +0 -105
- package/src/cli/runtime.ts +0 -7
- package/src/cli/safe-live.ts +0 -24
- package/src/cli/status-filter.ts +0 -114
- package/src/cli/util/api-context.ts +0 -85
- package/src/cli/version.ts +0 -8
- package/src/core/audit/persist.ts +0 -183
- package/src/core/checks/budget.ts +0 -59
- package/src/core/checks/checks/_crud-helpers.ts +0 -133
- package/src/core/checks/checks/_negative_mutator.ts +0 -133
- package/src/core/checks/checks/_readback-helpers.ts +0 -133
- package/src/core/checks/checks/content_type_conformance.ts +0 -39
- package/src/core/checks/checks/cross_call_references.ts +0 -147
- package/src/core/checks/checks/cursor_boundary_fuzzing.ts +0 -219
- package/src/core/checks/checks/ensure_resource_availability.ts +0 -62
- package/src/core/checks/checks/idempotency_replay.ts +0 -242
- package/src/core/checks/checks/ignored_auth.ts +0 -254
- package/src/core/checks/checks/index.ts +0 -68
- package/src/core/checks/checks/lifecycle_transitions.ts +0 -416
- package/src/core/checks/checks/missing_required_header.ts +0 -40
- package/src/core/checks/checks/negative_data_rejection.ts +0 -148
- package/src/core/checks/checks/not_a_server_error.ts +0 -35
- package/src/core/checks/checks/open_cors_on_sensitive.ts +0 -160
- package/src/core/checks/checks/pagination_invariants.ts +0 -419
- package/src/core/checks/checks/positive_data_acceptance.ts +0 -33
- package/src/core/checks/checks/rate_limit_headers_absent.ts +0 -77
- package/src/core/checks/checks/response_headers_conformance.ts +0 -74
- package/src/core/checks/checks/response_schema_conformance.ts +0 -30
- package/src/core/checks/checks/status_code_conformance.ts +0 -132
- package/src/core/checks/checks/unsupported_method.ts +0 -63
- package/src/core/checks/checks/use_after_free.ts +0 -78
- package/src/core/checks/index.ts +0 -30
- package/src/core/checks/mode.ts +0 -82
- package/src/core/checks/recommended-action.ts +0 -68
- package/src/core/checks/registry.ts +0 -78
- package/src/core/checks/runner.ts +0 -1461
- package/src/core/checks/sarif.ts +0 -230
- package/src/core/checks/spec-findings.ts +0 -308
- package/src/core/checks/stateful.ts +0 -121
- package/src/core/checks/types.ts +0 -305
- package/src/core/checks/zond-extensions.ts +0 -73
- package/src/core/classifier/recommended-action.ts +0 -251
- package/src/core/context/current.ts +0 -51
- package/src/core/context/session.ts +0 -78
- package/src/core/coverage/loader.ts +0 -216
- package/src/core/coverage/reasons.ts +0 -300
- package/src/core/diagnostics/db-analysis.ts +0 -666
- package/src/core/diagnostics/failure-class.ts +0 -140
- package/src/core/diagnostics/failure-hints.ts +0 -112
- package/src/core/diagnostics/spec-pointer.ts +0 -99
- package/src/core/diagnostics/suggested-fixes.ts +0 -155
- package/src/core/exporter/case-study/index.ts +0 -270
- package/src/core/exporter/curl.ts +0 -40
- package/src/core/exporter/exporter.ts +0 -48
- package/src/core/exporter/html-report/escape.ts +0 -24
- package/src/core/exporter/html-report/index.ts +0 -479
- package/src/core/exporter/html-report/script.ts +0 -100
- package/src/core/exporter/html-report/styles.ts +0 -408
- package/src/core/generator/catalog-builder.ts +0 -179
- package/src/core/generator/chunker.ts +0 -78
- package/src/core/generator/coverage-phase.ts +0 -0
- package/src/core/generator/coverage-scanner.ts +0 -87
- package/src/core/generator/data-factory.ts +0 -759
- package/src/core/generator/describe.ts +0 -302
- package/src/core/generator/endpoint-warnings.ts +0 -52
- package/src/core/generator/fixtures-builder.ts +0 -332
- package/src/core/generator/index.ts +0 -14
- package/src/core/generator/openapi-reader.ts +0 -297
- package/src/core/generator/path-param-disambig.ts +0 -140
- package/src/core/generator/resources-builder.ts +0 -898
- package/src/core/generator/schema-utils.ts +0 -112
- package/src/core/generator/serializer.ts +0 -343
- package/src/core/generator/suite-generator.ts +0 -1301
- package/src/core/generator/types.ts +0 -63
- package/src/core/identity/identity-file.ts +0 -0
- package/src/core/lint/affects.ts +0 -28
- package/src/core/lint/config.ts +0 -96
- package/src/core/lint/format.ts +0 -42
- package/src/core/lint/index.ts +0 -94
- package/src/core/lint/reporter.ts +0 -128
- package/src/core/lint/rules/consistency.ts +0 -158
- package/src/core/lint/rules/heuristics.ts +0 -97
- package/src/core/lint/rules/strictness.ts +0 -109
- package/src/core/lint/types.ts +0 -96
- package/src/core/lint/walker.ts +0 -248
- package/src/core/meta/meta-store.ts +0 -11
- package/src/core/output/README.md +0 -73
- package/src/core/output/index.ts +0 -13
- package/src/core/output/run.ts +0 -91
- package/src/core/output/types.ts +0 -122
- package/src/core/parser/dynamic-values.ts +0 -160
- package/src/core/parser/env-interpolation.ts +0 -104
- package/src/core/parser/filter.ts +0 -97
- package/src/core/parser/schema.ts +0 -359
- package/src/core/parser/types.ts +0 -117
- package/src/core/parser/variables.ts +0 -0
- package/src/core/parser/yaml-parser.ts +0 -181
- package/src/core/probe/bootstrap.ts +0 -34
- package/src/core/probe/dry-run-envelope.ts +0 -61
- package/src/core/probe/mass-assignment/classify.ts +0 -175
- package/src/core/probe/mass-assignment/cleanup.ts +0 -52
- package/src/core/probe/mass-assignment/digest.ts +0 -114
- package/src/core/probe/mass-assignment/orchestrator.ts +0 -459
- package/src/core/probe/mass-assignment/regression.ts +0 -141
- package/src/core/probe/mass-assignment/suspects.ts +0 -92
- package/src/core/probe/mass-assignment/types.ts +0 -135
- package/src/core/probe/mass-assignment-probe-class.ts +0 -198
- package/src/core/probe/mass-assignment-probe.ts +0 -27
- package/src/core/probe/mass-assignment-template.ts +0 -240
- package/src/core/probe/method-probe.ts +0 -164
- package/src/core/probe/method-shared.ts +0 -69
- package/src/core/probe/negative-probe.ts +0 -691
- package/src/core/probe/orphan-tracker.ts +0 -188
- package/src/core/probe/path-discovery.ts +0 -439
- package/src/core/probe/probe-harness.ts +0 -119
- package/src/core/probe/registry.ts +0 -89
- package/src/core/probe/runner.ts +0 -136
- package/src/core/probe/security/baseline.ts +0 -174
- package/src/core/probe/security/classify.ts +0 -341
- package/src/core/probe/security/cleanup.ts +0 -125
- package/src/core/probe/security/detectors.ts +0 -71
- package/src/core/probe/security/digest.ts +0 -104
- package/src/core/probe/security/orchestrator.ts +0 -398
- package/src/core/probe/security/regression.ts +0 -103
- package/src/core/probe/security/types.ts +0 -151
- package/src/core/probe/security-probe-class.ts +0 -207
- package/src/core/probe/security-probe.ts +0 -32
- package/src/core/probe/shared.ts +0 -531
- package/src/core/probe/static-probe-class.ts +0 -125
- package/src/core/probe/types.ts +0 -165
- package/src/core/probe/verdict-aggregator.ts +0 -33
- package/src/core/probe/webhooks-probe.ts +0 -282
- package/src/core/reporter/console.ts +0 -240
- package/src/core/reporter/index.ts +0 -22
- package/src/core/reporter/json.ts +0 -22
- package/src/core/reporter/junit.ts +0 -93
- package/src/core/reporter/ndjson.ts +0 -37
- package/src/core/reporter/types.ts +0 -15
- package/src/core/runner/assertions.ts +0 -383
- package/src/core/runner/async-pool.ts +0 -108
- package/src/core/runner/auth-path.ts +0 -8
- package/src/core/runner/ci-context.ts +0 -72
- package/src/core/runner/executor.ts +0 -652
- package/src/core/runner/expr-eval.ts +0 -41
- package/src/core/runner/form-encode.ts +0 -41
- package/src/core/runner/http-client.ts +0 -224
- package/src/core/runner/learn-drift.ts +0 -293
- package/src/core/runner/preflight-vars.ts +0 -153
- package/src/core/runner/progress-tracker.ts +0 -73
- package/src/core/runner/rate-limiter.ts +0 -185
- package/src/core/runner/run-kind.ts +0 -45
- package/src/core/runner/schema-validator.ts +0 -308
- package/src/core/runner/send-request.ts +0 -232
- package/src/core/runner/transforms.ts +0 -65
- package/src/core/runner/types.ts +0 -94
- package/src/core/secrets/registry.ts +0 -164
- package/src/core/secrets/secrets-file.ts +0 -115
- package/src/core/selectors/operation-filter.ts +0 -144
- package/src/core/setup-api.ts +0 -590
- package/src/core/severity/category.ts +0 -94
- package/src/core/severity/index.ts +0 -58
- package/src/core/spec/infer-schema.ts +0 -102
- package/src/core/spec/layers.ts +0 -154
- package/src/core/spec/merge-specs.ts +0 -156
- package/src/core/spec/schema-from-runs.ts +0 -117
- package/src/core/spec/schema-overlay.ts +0 -130
- package/src/core/util/ajv.ts +0 -13
- package/src/core/util/format-eta.ts +0 -21
- package/src/core/util/headers.ts +0 -9
- package/src/core/util/url.ts +0 -24
- package/src/core/utils.ts +0 -13
- package/src/core/workspace/config.ts +0 -129
- package/src/core/workspace/fixture-gap-report.ts +0 -84
- package/src/core/workspace/fixture-gaps.ts +0 -71
- package/src/core/workspace/manifest.ts +0 -283
- package/src/core/workspace/output-rotation.ts +0 -62
- package/src/core/workspace/root.ts +0 -96
- package/src/core/workspace/triage-path.ts +0 -87
- package/src/db/lint-runs.ts +0 -47
- package/src/db/migrate.ts +0 -128
- package/src/db/migrations/0001_run_kind.sql +0 -25
- package/src/db/migrations/0002_run_kind_request.sql +0 -59
- package/src/db/migrations/sql.d.ts +0 -4
- package/src/db/queries/collections.ts +0 -133
- package/src/db/queries/coverage.ts +0 -9
- package/src/db/queries/dashboard.ts +0 -59
- package/src/db/queries/results.ts +0 -216
- package/src/db/queries/runs.ts +0 -289
- package/src/db/queries/sessions.ts +0 -42
- package/src/db/queries/settings.ts +0 -28
- package/src/db/queries/types.ts +0 -172
- package/src/db/queries.ts +0 -75
- package/src/db/schema.ts +0 -298
|
@@ -1,666 +0,0 @@
|
|
|
1
|
-
import { getDb } from "../../db/schema.ts";
|
|
2
|
-
import { listCollections, listRuns, getRunById, getResultsByRunId, getCollectionById } from "../../db/queries.ts";
|
|
3
|
-
import { join } from "node:path";
|
|
4
|
-
import { classifyFailure, recommendedActionForGenerated, isGeneratedTest, type RecommendedAction } from "./failure-hints.ts";
|
|
5
|
-
import { buildSuggestedFixes, type SuggestedFix } from "./suggested-fixes.ts";
|
|
6
|
-
|
|
7
|
-
function truncateErrorMessage(raw: string | null | undefined, verbose?: boolean): string | undefined {
|
|
8
|
-
if (!raw) return undefined;
|
|
9
|
-
if (verbose || raw.length < 500) return raw;
|
|
10
|
-
const lines = raw.split(/\r?\n/);
|
|
11
|
-
const msgLines = [lines[0]!];
|
|
12
|
-
let traceCount = 0;
|
|
13
|
-
for (let i = 1; i < lines.length && traceCount < 3; i++) {
|
|
14
|
-
const line = lines[i]!;
|
|
15
|
-
if (/^\s+/.test(line) || /^\s*at\s/.test(line)) {
|
|
16
|
-
msgLines.push(line);
|
|
17
|
-
traceCount++;
|
|
18
|
-
}
|
|
19
|
-
}
|
|
20
|
-
const remaining = lines.length - msgLines.length;
|
|
21
|
-
if (remaining > 0) {
|
|
22
|
-
msgLines.push(`...[truncated ${remaining} lines]`);
|
|
23
|
-
}
|
|
24
|
-
return msgLines.join("\n");
|
|
25
|
-
}
|
|
26
|
-
|
|
27
|
-
/**
|
|
28
|
-
* ARV-305: build a short reason string for `by_recommended_action.examples`.
|
|
29
|
-
*
|
|
30
|
-
* Preference order:
|
|
31
|
-
* 1. trimmed `error_message` (assertion- or network-level message)
|
|
32
|
-
* 2. first failing assertion → `<field> <rule>: got <actual>`
|
|
33
|
-
* 3. undefined (nothing left to say)
|
|
34
|
-
*
|
|
35
|
-
* Without (2) every assertion-only failure leaves examples[].reason as
|
|
36
|
-
* undefined, so triage agents lose the one signal that lets them route
|
|
37
|
-
* past method/path/status (regenerate_suite vs tighten_validation
|
|
38
|
-
* collapse to identical-looking buckets).
|
|
39
|
-
*/
|
|
40
|
-
function buildExampleReason(
|
|
41
|
-
errorMessage: unknown,
|
|
42
|
-
assertions: unknown,
|
|
43
|
-
): string | undefined {
|
|
44
|
-
const trim = (s: string) => (s.length > 120 ? `${s.slice(0, 117)}...` : s);
|
|
45
|
-
if (typeof errorMessage === "string" && errorMessage.length > 0) {
|
|
46
|
-
return trim(errorMessage);
|
|
47
|
-
}
|
|
48
|
-
if (!Array.isArray(assertions)) return undefined;
|
|
49
|
-
for (const a of assertions) {
|
|
50
|
-
if (!a || typeof a !== "object") continue;
|
|
51
|
-
const row = a as Record<string, unknown>;
|
|
52
|
-
if (row.passed === false) {
|
|
53
|
-
const field = typeof row.field === "string" ? row.field : "";
|
|
54
|
-
const rule = typeof row.rule === "string" ? row.rule : "";
|
|
55
|
-
const actual = "actual" in row ? row.actual : undefined;
|
|
56
|
-
const actualStr = actual === undefined || actual === null
|
|
57
|
-
? ""
|
|
58
|
-
: typeof actual === "string"
|
|
59
|
-
? actual
|
|
60
|
-
: JSON.stringify(actual);
|
|
61
|
-
const parts = [field, rule].filter(Boolean).join(" ");
|
|
62
|
-
const text = actualStr ? `${parts}: got ${actualStr}` : parts;
|
|
63
|
-
return text ? trim(text) : undefined;
|
|
64
|
-
}
|
|
65
|
-
}
|
|
66
|
-
return undefined;
|
|
67
|
-
}
|
|
68
|
-
|
|
69
|
-
function parseBodySafe(raw: string | null | undefined): unknown {
|
|
70
|
-
if (!raw) return undefined;
|
|
71
|
-
const truncated = raw.length > 2000 ? raw.slice(0, 2000) + "\u2026[truncated]" : raw;
|
|
72
|
-
try {
|
|
73
|
-
return JSON.parse(raw);
|
|
74
|
-
} catch {
|
|
75
|
-
return truncated;
|
|
76
|
-
}
|
|
77
|
-
}
|
|
78
|
-
|
|
79
|
-
const USEFUL_HEADERS = new Set([
|
|
80
|
-
"content-type", "content-length", "location", "retry-after",
|
|
81
|
-
"www-authenticate", "allow",
|
|
82
|
-
]);
|
|
83
|
-
const USEFUL_PREFIXES = ["x-", "ratelimit"];
|
|
84
|
-
|
|
85
|
-
/** ARV-103 (F8): true when at least one assertion on the failing step is
|
|
86
|
-
* a schema-validation kind. `--validate-schema` annotates each violated
|
|
87
|
-
* field with `kind: "schema"` (set in src/core/runner/schema-validator.ts).
|
|
88
|
-
* The assertions column is stored as JSON in SQLite; parse defensively. */
|
|
89
|
-
function hasSchemaAssertion(raw: string | unknown[] | null | undefined): boolean {
|
|
90
|
-
if (raw === null || raw === undefined) return false;
|
|
91
|
-
let arr: unknown[];
|
|
92
|
-
if (Array.isArray(raw)) {
|
|
93
|
-
arr = raw;
|
|
94
|
-
} else if (typeof raw === "string") {
|
|
95
|
-
try {
|
|
96
|
-
const parsed = JSON.parse(raw);
|
|
97
|
-
if (!Array.isArray(parsed)) return false;
|
|
98
|
-
arr = parsed;
|
|
99
|
-
} catch {
|
|
100
|
-
return false;
|
|
101
|
-
}
|
|
102
|
-
} else {
|
|
103
|
-
return false;
|
|
104
|
-
}
|
|
105
|
-
for (const a of arr) {
|
|
106
|
-
if (a && typeof a === "object" && (a as { kind?: unknown }).kind === "schema") return true;
|
|
107
|
-
}
|
|
108
|
-
return false;
|
|
109
|
-
}
|
|
110
|
-
|
|
111
|
-
function filterHeaders(raw: string | null | undefined): Record<string, string> | undefined {
|
|
112
|
-
if (!raw) return undefined;
|
|
113
|
-
try {
|
|
114
|
-
const h = JSON.parse(raw) as Record<string, string>;
|
|
115
|
-
const out: Record<string, string> = {};
|
|
116
|
-
for (const [k, v] of Object.entries(h)) {
|
|
117
|
-
const l = k.toLowerCase();
|
|
118
|
-
if (USEFUL_HEADERS.has(l) || USEFUL_PREFIXES.some(p => l.startsWith(p))) {
|
|
119
|
-
out[k] = v;
|
|
120
|
-
}
|
|
121
|
-
}
|
|
122
|
-
return Object.keys(out).length > 0 ? out : undefined;
|
|
123
|
-
} catch { return undefined; }
|
|
124
|
-
}
|
|
125
|
-
|
|
126
|
-
export interface RunDetail {
|
|
127
|
-
run: {
|
|
128
|
-
id: number;
|
|
129
|
-
started_at: string;
|
|
130
|
-
finished_at: string | null;
|
|
131
|
-
total: number;
|
|
132
|
-
passed: number;
|
|
133
|
-
failed: number;
|
|
134
|
-
skipped: number;
|
|
135
|
-
trigger: string | null;
|
|
136
|
-
environment: string | null;
|
|
137
|
-
duration_ms: number | null;
|
|
138
|
-
};
|
|
139
|
-
results: Array<{
|
|
140
|
-
suite_name: string;
|
|
141
|
-
test_name: string;
|
|
142
|
-
status: string;
|
|
143
|
-
duration_ms: number | null;
|
|
144
|
-
request_method: string | null;
|
|
145
|
-
request_url: string | null;
|
|
146
|
-
response_status: number | null;
|
|
147
|
-
error_message?: string;
|
|
148
|
-
assertions: unknown;
|
|
149
|
-
}>;
|
|
150
|
-
}
|
|
151
|
-
|
|
152
|
-
export function getRunDetail(runId: number, verbose?: boolean, dbPath?: string): RunDetail {
|
|
153
|
-
getDb(dbPath);
|
|
154
|
-
const run = getRunById(runId);
|
|
155
|
-
if (!run) throw new Error(`Run ${runId} not found`);
|
|
156
|
-
const results = getResultsByRunId(runId);
|
|
157
|
-
return {
|
|
158
|
-
run: {
|
|
159
|
-
id: run.id,
|
|
160
|
-
started_at: run.started_at,
|
|
161
|
-
finished_at: run.finished_at,
|
|
162
|
-
total: run.total,
|
|
163
|
-
passed: run.passed,
|
|
164
|
-
failed: run.failed,
|
|
165
|
-
skipped: run.skipped,
|
|
166
|
-
trigger: run.trigger,
|
|
167
|
-
environment: run.environment,
|
|
168
|
-
duration_ms: run.duration_ms,
|
|
169
|
-
},
|
|
170
|
-
results: results.map(r => ({
|
|
171
|
-
suite_name: r.suite_name,
|
|
172
|
-
test_name: r.test_name,
|
|
173
|
-
status: r.status,
|
|
174
|
-
duration_ms: r.duration_ms,
|
|
175
|
-
request_method: r.request_method,
|
|
176
|
-
request_url: r.request_url,
|
|
177
|
-
response_status: r.response_status,
|
|
178
|
-
error_message: truncateErrorMessage(r.error_message, verbose),
|
|
179
|
-
assertions: r.assertions,
|
|
180
|
-
})),
|
|
181
|
-
};
|
|
182
|
-
}
|
|
183
|
-
|
|
184
|
-
export interface FailureGroup {
|
|
185
|
-
pattern: string;
|
|
186
|
-
count: number;
|
|
187
|
-
failure_type: string;
|
|
188
|
-
recommended_action: RecommendedAction;
|
|
189
|
-
examples: string[];
|
|
190
|
-
response_status: number | null;
|
|
191
|
-
}
|
|
192
|
-
|
|
193
|
-
export interface CascadeSkipGroup {
|
|
194
|
-
capture_var: string;
|
|
195
|
-
count: number;
|
|
196
|
-
examples: string[];
|
|
197
|
-
}
|
|
198
|
-
|
|
199
|
-
export interface DiagnoseResult {
|
|
200
|
-
run: {
|
|
201
|
-
id: number;
|
|
202
|
-
started_at: string;
|
|
203
|
-
environment: string | null;
|
|
204
|
-
duration_ms: number | null;
|
|
205
|
-
};
|
|
206
|
-
summary: {
|
|
207
|
-
total: number;
|
|
208
|
-
passed: number;
|
|
209
|
-
failed: number;
|
|
210
|
-
api_errors: number;
|
|
211
|
-
assertion_failures: number;
|
|
212
|
-
network_errors: number;
|
|
213
|
-
};
|
|
214
|
-
cascade_skips?: CascadeSkipGroup[];
|
|
215
|
-
/** TASK-29: actionable suggestions populated from 404 placeholder
|
|
216
|
-
* detection + .env.yaml unfilled-key audit. Empty / undefined when
|
|
217
|
-
* nothing actionable was found. */
|
|
218
|
-
suggested_fixes?: SuggestedFix[];
|
|
219
|
-
failures: Array<{
|
|
220
|
-
suite_name: string;
|
|
221
|
-
test_name: string;
|
|
222
|
-
suite_file?: string;
|
|
223
|
-
status: string;
|
|
224
|
-
failure_type: string;
|
|
225
|
-
recommended_action: RecommendedAction;
|
|
226
|
-
error_message?: string;
|
|
227
|
-
request_method: string | null;
|
|
228
|
-
request_url: string | null;
|
|
229
|
-
response_status: number | null;
|
|
230
|
-
response_body?: unknown;
|
|
231
|
-
response_headers?: Record<string, string>;
|
|
232
|
-
assertions: unknown;
|
|
233
|
-
duration_ms: number | null;
|
|
234
|
-
/** ARV-159: when this entry is the representative of a collapsed group
|
|
235
|
-
* (status|failure_type signature), the total size of that group. Lets
|
|
236
|
-
* consumers reading `.data.failures[]` see "this signature stands for
|
|
237
|
-
* N underlying tests" without cross-referencing `.grouped_failures[]`.
|
|
238
|
-
* Omitted when no collapsing occurred (failures ≤ 5 or
|
|
239
|
-
* --verbose). */
|
|
240
|
-
group_count?: number;
|
|
241
|
-
}>;
|
|
242
|
-
grouped_failures?: FailureGroup[];
|
|
243
|
-
/** ARV-101 (F6): top-level aggregation keyed by `recommended_action`
|
|
244
|
-
* enum so triage agents (zond-triage skill) can route on the canonical
|
|
245
|
-
* action without re-folding `failures[].recommended_action` through
|
|
246
|
-
* `jq | group_by`. Built from the *full* failure set (not the compact
|
|
247
|
-
* subset), so counts match `.summary.failed`. Each bucket carries
|
|
248
|
-
* total count + a small examples list (`<suite>/<test>`). Empty when
|
|
249
|
-
* there are no failures. */
|
|
250
|
-
by_recommended_action?: Record<string, {
|
|
251
|
-
count: number;
|
|
252
|
-
/** ARV-228: each example carries the per-failure context the
|
|
253
|
-
* zond-triage skill renders in its output template (`POST
|
|
254
|
-
* /v1/projects → 500 (×3) — <reason>`). Previously a bare
|
|
255
|
-
* `string[]` of `<suite>/<test>` ids — agents had to cross-join
|
|
256
|
-
* with `failures[]` to recover method/path/status, which broke
|
|
257
|
-
* triage scripts on large runs. Bounded to 5 entries per bucket
|
|
258
|
-
* (same cap as the legacy string form). */
|
|
259
|
-
examples: Array<{
|
|
260
|
-
suite: string;
|
|
261
|
-
test: string;
|
|
262
|
-
method: string;
|
|
263
|
-
path: string;
|
|
264
|
-
status: number;
|
|
265
|
-
reason?: string;
|
|
266
|
-
}>;
|
|
267
|
-
}>;
|
|
268
|
-
}
|
|
269
|
-
|
|
270
|
-
export function diagnoseRun(runId: number, verbose?: boolean, dbPath?: string, maxExamples?: number): DiagnoseResult {
|
|
271
|
-
getDb(dbPath);
|
|
272
|
-
const diagRun = getRunById(runId);
|
|
273
|
-
if (!diagRun) throw new Error(`Run ${runId} not found`);
|
|
274
|
-
|
|
275
|
-
let envFilePath: string | undefined;
|
|
276
|
-
if (diagRun.collection_id) {
|
|
277
|
-
const collection = getCollectionById(diagRun.collection_id);
|
|
278
|
-
if (collection?.base_dir) {
|
|
279
|
-
envFilePath = join(collection.base_dir, ".env.yaml").replace(/\\/g, "/");
|
|
280
|
-
}
|
|
281
|
-
}
|
|
282
|
-
|
|
283
|
-
const allResults = getResultsByRunId(runId);
|
|
284
|
-
const failures = allResults
|
|
285
|
-
.filter(r => r.status === "fail" || r.status === "error")
|
|
286
|
-
.map(r => {
|
|
287
|
-
const parsedBody = parseBodySafe(r.response_body);
|
|
288
|
-
const failure_type = classifyFailure(r.status, r.response_status);
|
|
289
|
-
// ARV-42: generator-emitted suites should not route to fix_test_logic —
|
|
290
|
-
// editing the YAML gets clobbered on the next `zond audit`.
|
|
291
|
-
const generated = isGeneratedTest(r.provenance, r.suite_file);
|
|
292
|
-
// ARV-103 (F8): walk the assertions array to detect a schema-kind
|
|
293
|
-
// failure (--validate-schema annotates each assertion with its kind).
|
|
294
|
-
// When present, propagate the flag so the classifier routes to
|
|
295
|
-
// report_backend_bug — schema violations are real contract bugs, not
|
|
296
|
-
// test-logic mistakes.
|
|
297
|
-
const schema_violation = hasSchemaAssertion(r.assertions);
|
|
298
|
-
const rec_action = recommendedActionForGenerated(failure_type, r.response_status, generated, schema_violation);
|
|
299
|
-
return {
|
|
300
|
-
suite_name: r.suite_name,
|
|
301
|
-
test_name: r.test_name,
|
|
302
|
-
...(r.suite_file ? { suite_file: r.suite_file } : {}),
|
|
303
|
-
status: r.status,
|
|
304
|
-
failure_type,
|
|
305
|
-
recommended_action: rec_action,
|
|
306
|
-
error_message: truncateErrorMessage(r.error_message, verbose),
|
|
307
|
-
request_method: r.request_method,
|
|
308
|
-
request_url: r.request_url,
|
|
309
|
-
response_status: r.response_status,
|
|
310
|
-
response_body: parsedBody,
|
|
311
|
-
response_headers: filterHeaders(r.response_headers),
|
|
312
|
-
assertions: r.assertions,
|
|
313
|
-
duration_ms: r.duration_ms,
|
|
314
|
-
};
|
|
315
|
-
});
|
|
316
|
-
|
|
317
|
-
let apiErrors = 0, assertionFailures = 0, networkErrors = 0;
|
|
318
|
-
for (const f of failures) {
|
|
319
|
-
if (f.failure_type === "api_error") apiErrors++;
|
|
320
|
-
else if (f.failure_type === "assertion_failed") assertionFailures++;
|
|
321
|
-
else if (f.failure_type === "network_error") networkErrors++;
|
|
322
|
-
}
|
|
323
|
-
|
|
324
|
-
// Cascade skips: skipped tests due to missing captures from failed create steps
|
|
325
|
-
const CASCADE_RE = /^Depends on missing capture: (.+)$/;
|
|
326
|
-
const groupMap = new Map<string, string[]>();
|
|
327
|
-
for (const r of allResults) {
|
|
328
|
-
if (r.status !== "skip") continue;
|
|
329
|
-
const match = CASCADE_RE.exec(r.error_message ?? "");
|
|
330
|
-
if (!match) continue;
|
|
331
|
-
const captureVar = match[1]!;
|
|
332
|
-
const existing = groupMap.get(captureVar) ?? [];
|
|
333
|
-
existing.push(`${r.suite_name}/${r.test_name}`);
|
|
334
|
-
groupMap.set(captureVar, existing);
|
|
335
|
-
}
|
|
336
|
-
const cascade_skips: CascadeSkipGroup[] | undefined = groupMap.size > 0
|
|
337
|
-
? [...groupMap.entries()].map(([capture_var, examples]) => ({
|
|
338
|
-
capture_var,
|
|
339
|
-
count: examples.length,
|
|
340
|
-
examples: examples.slice(0, 3),
|
|
341
|
-
}))
|
|
342
|
-
: undefined;
|
|
343
|
-
|
|
344
|
-
const { grouped_failures, compactFailures } = verbose
|
|
345
|
-
? { grouped_failures: undefined, compactFailures: failures }
|
|
346
|
-
: groupFailures(failures, maxExamples);
|
|
347
|
-
|
|
348
|
-
// TASK-29: surface placeholder path-params + unfilled .env.yaml keys.
|
|
349
|
-
const suggestedFixes = buildSuggestedFixes({
|
|
350
|
-
failures: failures.map(f => ({
|
|
351
|
-
response_status: f.response_status,
|
|
352
|
-
request_url: f.request_url,
|
|
353
|
-
suite_name: f.suite_name,
|
|
354
|
-
test_name: f.test_name,
|
|
355
|
-
})),
|
|
356
|
-
envFilePath,
|
|
357
|
-
});
|
|
358
|
-
|
|
359
|
-
// ARV-101 (F6): aggregate failures by recommended_action enum so triage
|
|
360
|
-
// agents read .data.by_recommended_action.fix_env.count instead of
|
|
361
|
-
// re-folding failures[].recommended_action through `jq | group_by`. Built
|
|
362
|
-
// from the full failure set (not compactFailures) so counts match
|
|
363
|
-
// .summary.failed. Bounded examples list (5) keeps payload small while
|
|
364
|
-
// still pointing at concrete suites the agent can open.
|
|
365
|
-
//
|
|
366
|
-
// ARV-228: each example is now an object carrying method/path/status/
|
|
367
|
-
// reason so the zond-triage skill can render its output template
|
|
368
|
-
// ("POST /v1/projects → 500 (×3) — TypeError") without cross-joining
|
|
369
|
-
// failures[]. Bound preserved at 5/bucket; ordering = insertion order
|
|
370
|
-
// (matches failures[] traversal, deterministic per run).
|
|
371
|
-
const by_recommended_action: Record<string, NonNullable<DiagnoseResult["by_recommended_action"]>[string]> = {};
|
|
372
|
-
for (const f of failures) {
|
|
373
|
-
const key = f.recommended_action;
|
|
374
|
-
let bucket = by_recommended_action[key];
|
|
375
|
-
if (!bucket) {
|
|
376
|
-
bucket = { count: 0, examples: [] };
|
|
377
|
-
by_recommended_action[key] = bucket;
|
|
378
|
-
}
|
|
379
|
-
bucket.count += 1;
|
|
380
|
-
if (bucket.examples.length < 5) {
|
|
381
|
-
// Trim reason to keep the bucket compact — full error_message lives
|
|
382
|
-
// in failures[].error_message for agents that want it.
|
|
383
|
-
//
|
|
384
|
-
// ARV-305: when there is no top-level error_message (typical for
|
|
385
|
-
// assertion failures — the row carries the failing rule in
|
|
386
|
-
// .assertions, not a free-form message), build a reason out of
|
|
387
|
-
// the first failing assertion so the example is not stripped
|
|
388
|
-
// down to method/path/status. The fallback keeps the field
|
|
389
|
-
// populated whenever any failure context exists.
|
|
390
|
-
const reason = buildExampleReason(f.error_message, f.assertions);
|
|
391
|
-
bucket.examples.push({
|
|
392
|
-
suite: f.suite_name,
|
|
393
|
-
test: f.test_name,
|
|
394
|
-
// DB columns are nullable (early steps may not have a request
|
|
395
|
-
// recorded yet — e.g. fixture-load failures). Coerce to "" / 0
|
|
396
|
-
// rather than leaking null into the contract.
|
|
397
|
-
method: f.request_method ?? "",
|
|
398
|
-
path: f.request_url ?? "",
|
|
399
|
-
status: f.response_status ?? 0,
|
|
400
|
-
...(reason ? { reason } : {}),
|
|
401
|
-
});
|
|
402
|
-
}
|
|
403
|
-
}
|
|
404
|
-
|
|
405
|
-
return {
|
|
406
|
-
run: {
|
|
407
|
-
id: diagRun.id,
|
|
408
|
-
started_at: diagRun.started_at,
|
|
409
|
-
environment: diagRun.environment,
|
|
410
|
-
duration_ms: diagRun.duration_ms,
|
|
411
|
-
},
|
|
412
|
-
summary: {
|
|
413
|
-
total: diagRun.total,
|
|
414
|
-
passed: diagRun.passed,
|
|
415
|
-
failed: diagRun.failed,
|
|
416
|
-
api_errors: apiErrors,
|
|
417
|
-
assertion_failures: assertionFailures,
|
|
418
|
-
network_errors: networkErrors,
|
|
419
|
-
},
|
|
420
|
-
...(cascade_skips ? { cascade_skips } : {}),
|
|
421
|
-
...(suggestedFixes.length > 0 ? { suggested_fixes: suggestedFixes } : {}),
|
|
422
|
-
failures: compactFailures,
|
|
423
|
-
...(grouped_failures ? { grouped_failures } : {}),
|
|
424
|
-
...(failures.length > 0 ? { by_recommended_action } : {}),
|
|
425
|
-
};
|
|
426
|
-
}
|
|
427
|
-
|
|
428
|
-
type FailureItem = { suite_name: string; test_name: string; failure_type: string; recommended_action: RecommendedAction; response_status: number | null; group_count?: number };
|
|
429
|
-
|
|
430
|
-
/** Group similar failures for compact output. Exported for testing. */
|
|
431
|
-
export function groupFailures<T extends FailureItem>(failures: T[], maxExamples = 2): { grouped_failures?: FailureGroup[]; compactFailures: T[] } {
|
|
432
|
-
if (failures.length <= 5) {
|
|
433
|
-
return { compactFailures: failures };
|
|
434
|
-
}
|
|
435
|
-
|
|
436
|
-
const groupMap = new Map<string, { items: T[]; failure_type: string; response_status: number | null }>();
|
|
437
|
-
|
|
438
|
-
for (const f of failures) {
|
|
439
|
-
const key = `${f.response_status ?? "null"}|${f.failure_type}`;
|
|
440
|
-
const existing = groupMap.get(key);
|
|
441
|
-
if (existing) {
|
|
442
|
-
existing.items.push(f);
|
|
443
|
-
} else {
|
|
444
|
-
groupMap.set(key, {
|
|
445
|
-
items: [f],
|
|
446
|
-
failure_type: f.failure_type,
|
|
447
|
-
response_status: f.response_status,
|
|
448
|
-
});
|
|
449
|
-
}
|
|
450
|
-
}
|
|
451
|
-
|
|
452
|
-
const hasGroups = [...groupMap.values()].some(g => g.items.length > 2);
|
|
453
|
-
if (!hasGroups) {
|
|
454
|
-
return { compactFailures: failures };
|
|
455
|
-
}
|
|
456
|
-
|
|
457
|
-
const grouped_failures: FailureGroup[] = [];
|
|
458
|
-
const compactFailures: T[] = [];
|
|
459
|
-
|
|
460
|
-
for (const [, group] of groupMap) {
|
|
461
|
-
const pattern = group.response_status
|
|
462
|
-
? `${group.response_status} ${group.failure_type}`
|
|
463
|
-
: group.failure_type;
|
|
464
|
-
// 5xx (api_error) are critical backend bugs — never collapse them.
|
|
465
|
-
// Diagnose must surface every 5xx, otherwise users miss real
|
|
466
|
-
// regressions hidden behind a single sample.
|
|
467
|
-
const isApiError = group.failure_type === "api_error";
|
|
468
|
-
const showAll = isApiError || maxExamples === 0;
|
|
469
|
-
grouped_failures.push({
|
|
470
|
-
pattern,
|
|
471
|
-
count: group.items.length,
|
|
472
|
-
failure_type: group.failure_type,
|
|
473
|
-
recommended_action: group.items[0]!.recommended_action,
|
|
474
|
-
examples: (showAll ? group.items : group.items.slice(0, maxExamples)).map(f => `${f.suite_name}/${f.test_name}`),
|
|
475
|
-
response_status: group.response_status,
|
|
476
|
-
});
|
|
477
|
-
if (isApiError) {
|
|
478
|
-
compactFailures.push(...group.items);
|
|
479
|
-
} else {
|
|
480
|
-
// ARV-159: tag the representative with the group size so
|
|
481
|
-
// `.data.failures[]` carries the multiplier inline.
|
|
482
|
-
const rep = { ...group.items[0]!, group_count: group.items.length };
|
|
483
|
-
compactFailures.push(rep as T);
|
|
484
|
-
}
|
|
485
|
-
}
|
|
486
|
-
|
|
487
|
-
return { grouped_failures, compactFailures };
|
|
488
|
-
}
|
|
489
|
-
|
|
490
|
-
export interface BodyFieldChange {
|
|
491
|
-
field: string;
|
|
492
|
-
change: "added" | "removed" | "type_changed";
|
|
493
|
-
before?: string;
|
|
494
|
-
after?: string;
|
|
495
|
-
/** ARV-352: structural scope of the change, derived deterministically from
|
|
496
|
-
* the path. `element` = the path crosses an array boundary (`[]`), i.e.
|
|
497
|
-
* it's a field of a *collection item* — on list/log endpoints two samplings
|
|
498
|
-
* return DIFFERENT objects, so element-level added/removed/type_changed is
|
|
499
|
-
* schema-of-union variance across the sampled set, not a contract move.
|
|
500
|
-
* `container` = no `[]` in the path — the response envelope/pagination
|
|
501
|
-
* skeleton, where a change IS real drift. NOT a suppression heuristic
|
|
502
|
-
* (ARV-337): nothing is dropped or down-ranked; the agent judges using the
|
|
503
|
-
* scope tag + endpoint context. */
|
|
504
|
-
scope: "container" | "element";
|
|
505
|
-
}
|
|
506
|
-
|
|
507
|
-
export interface BodyDiff {
|
|
508
|
-
suite: string;
|
|
509
|
-
test: string;
|
|
510
|
-
changes: BodyFieldChange[];
|
|
511
|
-
}
|
|
512
|
-
|
|
513
|
-
export interface CompareResult {
|
|
514
|
-
runA: { id: number; started_at: string };
|
|
515
|
-
runB: { id: number; started_at: string };
|
|
516
|
-
summary: {
|
|
517
|
-
regressions: number;
|
|
518
|
-
fixes: number;
|
|
519
|
-
unchanged: number;
|
|
520
|
-
newTests: number;
|
|
521
|
-
removedTests: number;
|
|
522
|
-
bodyChanges: number;
|
|
523
|
-
/** ARV-352: of `bodyChanges`, how many touch the response envelope
|
|
524
|
-
* (`container`) vs collection-item fields (`element`). Element-heavy diffs
|
|
525
|
-
* on list/log endpoints are schema-of-union variance across a re-sampled
|
|
526
|
-
* set, not contract drift — split so triage doesn't read them as regression. */
|
|
527
|
-
bodyChangesContainer: number;
|
|
528
|
-
bodyChangesElement: number;
|
|
529
|
-
};
|
|
530
|
-
regressions: Array<{ suite: string; test: string; before: string; after: string }>;
|
|
531
|
-
fixes: Array<{ suite: string; test: string; before: string; after: string }>;
|
|
532
|
-
/** ARV-339: field-level response-shape diff for tests present in both runs.
|
|
533
|
-
* Status-diff answers "what broke"; this answers "how the contract moved". */
|
|
534
|
-
body_changes: BodyDiff[];
|
|
535
|
-
hasRegressions: boolean;
|
|
536
|
-
}
|
|
537
|
-
|
|
538
|
-
/** ARV-339: flatten a parsed JSON body into `path → union of leaf types`.
|
|
539
|
-
* Array elements collapse under `[]` so item count/order don't add noise. */
|
|
540
|
-
function bodyShape(value: unknown, path: string, out: Map<string, Set<string>>): void {
|
|
541
|
-
if (Array.isArray(value)) {
|
|
542
|
-
if (value.length === 0) addShape(out, path, "array");
|
|
543
|
-
for (const item of value) bodyShape(item, `${path}[]`, out);
|
|
544
|
-
} else if (value !== null && typeof value === "object") {
|
|
545
|
-
const entries = Object.entries(value as Record<string, unknown>);
|
|
546
|
-
if (entries.length === 0) addShape(out, path, "object");
|
|
547
|
-
for (const [k, v] of entries) bodyShape(v, path ? `${path}.${k}` : k, out);
|
|
548
|
-
} else {
|
|
549
|
-
addShape(out, path, value === null ? "null" : typeof value);
|
|
550
|
-
}
|
|
551
|
-
}
|
|
552
|
-
|
|
553
|
-
function addShape(out: Map<string, Set<string>>, path: string, type: string): void {
|
|
554
|
-
const key = path || "$";
|
|
555
|
-
const set = out.get(key) ?? new Set<string>();
|
|
556
|
-
set.add(type);
|
|
557
|
-
out.set(key, set);
|
|
558
|
-
}
|
|
559
|
-
|
|
560
|
-
const typeLabel = (s: Set<string>): string => [...s].sort().join("|");
|
|
561
|
-
|
|
562
|
-
/** ARV-339: diff two stored response bodies at field level. Returns [] when
|
|
563
|
-
* either side is missing / non-JSON — a shape diff of prose is meaningless. */
|
|
564
|
-
export function diffBodyShapes(rawA: string | null, rawB: string | null): BodyFieldChange[] {
|
|
565
|
-
if (!rawA || !rawB || rawA === rawB) return [];
|
|
566
|
-
let a: unknown, b: unknown;
|
|
567
|
-
try { a = JSON.parse(rawA); b = JSON.parse(rawB); } catch { return []; }
|
|
568
|
-
if (typeof a !== "object" || a === null || typeof b !== "object" || b === null) return [];
|
|
569
|
-
const shapeA = new Map<string, Set<string>>();
|
|
570
|
-
const shapeB = new Map<string, Set<string>>();
|
|
571
|
-
bodyShape(a, "", shapeA);
|
|
572
|
-
bodyShape(b, "", shapeB);
|
|
573
|
-
const scopeOf = (field: string): "container" | "element" =>
|
|
574
|
-
field.includes("[]") ? "element" : "container";
|
|
575
|
-
const changes: BodyFieldChange[] = [];
|
|
576
|
-
for (const [field, typesB] of shapeB) {
|
|
577
|
-
const typesA = shapeA.get(field);
|
|
578
|
-
if (!typesA) changes.push({ field, change: "added", after: typeLabel(typesB), scope: scopeOf(field) });
|
|
579
|
-
else if (typeLabel(typesA) !== typeLabel(typesB)) {
|
|
580
|
-
changes.push({ field, change: "type_changed", before: typeLabel(typesA), after: typeLabel(typesB), scope: scopeOf(field) });
|
|
581
|
-
}
|
|
582
|
-
}
|
|
583
|
-
for (const [field, typesA] of shapeA) {
|
|
584
|
-
if (!shapeB.has(field)) changes.push({ field, change: "removed", before: typeLabel(typesA), scope: scopeOf(field) });
|
|
585
|
-
}
|
|
586
|
-
return changes.sort((x, y) => x.field.localeCompare(y.field));
|
|
587
|
-
}
|
|
588
|
-
|
|
589
|
-
export function compareRuns(idA: number, idB: number, dbPath?: string): CompareResult {
|
|
590
|
-
getDb(dbPath);
|
|
591
|
-
const runARecord = getRunById(idA);
|
|
592
|
-
const runBRecord = getRunById(idB);
|
|
593
|
-
if (!runARecord) throw new Error(`Run #${idA} not found`);
|
|
594
|
-
if (!runBRecord) throw new Error(`Run #${idB} not found`);
|
|
595
|
-
|
|
596
|
-
const resultsA = getResultsByRunId(idA);
|
|
597
|
-
const resultsB = getResultsByRunId(idB);
|
|
598
|
-
|
|
599
|
-
const mapA = new Map<string, string>();
|
|
600
|
-
const mapB = new Map<string, string>();
|
|
601
|
-
const bodyA = new Map<string, string | null>();
|
|
602
|
-
const bodyB = new Map<string, string | null>();
|
|
603
|
-
for (const r of resultsA) {
|
|
604
|
-
mapA.set(`${r.suite_name}::${r.test_name}`, r.status);
|
|
605
|
-
bodyA.set(`${r.suite_name}::${r.test_name}`, r.response_body);
|
|
606
|
-
}
|
|
607
|
-
for (const r of resultsB) {
|
|
608
|
-
mapB.set(`${r.suite_name}::${r.test_name}`, r.status);
|
|
609
|
-
bodyB.set(`${r.suite_name}::${r.test_name}`, r.response_body);
|
|
610
|
-
}
|
|
611
|
-
|
|
612
|
-
const regressions: Array<{ suite: string; test: string; before: string; after: string }> = [];
|
|
613
|
-
const fixes: Array<{ suite: string; test: string; before: string; after: string }> = [];
|
|
614
|
-
let unchanged = 0;
|
|
615
|
-
let newTests = 0;
|
|
616
|
-
let removedTests = 0;
|
|
617
|
-
|
|
618
|
-
const body_changes: BodyDiff[] = [];
|
|
619
|
-
|
|
620
|
-
for (const [key, statusB] of mapB) {
|
|
621
|
-
const statusA = mapA.get(key);
|
|
622
|
-
if (statusA === undefined) { newTests++; continue; }
|
|
623
|
-
const [suite, test] = key.split("::") as [string, string];
|
|
624
|
-
const changes = diffBodyShapes(bodyA.get(key) ?? null, bodyB.get(key) ?? null);
|
|
625
|
-
if (changes.length > 0) body_changes.push({ suite, test, changes });
|
|
626
|
-
const wasPass = statusA === "pass";
|
|
627
|
-
const isPass = statusB === "pass";
|
|
628
|
-
const wasFail = statusA === "fail" || statusA === "error";
|
|
629
|
-
const isFail = statusB === "fail" || statusB === "error";
|
|
630
|
-
if (wasPass && isFail) regressions.push({ suite, test, before: statusA, after: statusB });
|
|
631
|
-
else if (wasFail && isPass) fixes.push({ suite, test, before: statusA, after: statusB });
|
|
632
|
-
else unchanged++;
|
|
633
|
-
}
|
|
634
|
-
for (const key of mapA.keys()) {
|
|
635
|
-
if (!mapB.has(key)) removedTests++;
|
|
636
|
-
}
|
|
637
|
-
|
|
638
|
-
return {
|
|
639
|
-
runA: { id: idA, started_at: runARecord.started_at },
|
|
640
|
-
runB: { id: idB, started_at: runBRecord.started_at },
|
|
641
|
-
summary: {
|
|
642
|
-
regressions: regressions.length,
|
|
643
|
-
fixes: fixes.length,
|
|
644
|
-
unchanged,
|
|
645
|
-
newTests,
|
|
646
|
-
removedTests,
|
|
647
|
-
bodyChanges: body_changes.length,
|
|
648
|
-
bodyChangesContainer: body_changes.filter(d => d.changes.some(c => c.scope === "container")).length,
|
|
649
|
-
bodyChangesElement: body_changes.filter(d => d.changes.every(c => c.scope === "element")).length,
|
|
650
|
-
},
|
|
651
|
-
regressions,
|
|
652
|
-
fixes,
|
|
653
|
-
body_changes,
|
|
654
|
-
hasRegressions: regressions.length > 0,
|
|
655
|
-
};
|
|
656
|
-
}
|
|
657
|
-
|
|
658
|
-
export function getCollections(dbPath?: string) {
|
|
659
|
-
getDb(dbPath);
|
|
660
|
-
return listCollections();
|
|
661
|
-
}
|
|
662
|
-
|
|
663
|
-
export function getRuns(limit?: number, dbPath?: string) {
|
|
664
|
-
getDb(dbPath);
|
|
665
|
-
return listRuns(limit ?? 20);
|
|
666
|
-
}
|