@nasebanal/cli 0.1.0 → 0.3.0

package/dist/reports/locust.js

@@ -0,0 +1,68 @@
+/**
+ * Locust's CSV output (`--csv <prefix>`) -> the normalised `load` summary.
+ *
+ * The pass/fail rule is the one nb-quickstarts' `locust/bin/exit_code.py`
+ * applies to Locust's own exit code, reproduced from the files so a report can
+ * be uploaded after the run without carrying the exit code along: any unhandled
+ * task exception, or a failure ratio above the allowed one, fails the run.
+ */
+import { toRecords } from "./csv.js";
+function num(value) {
+    if (value === undefined || value === "" || value === "N/A")
+        return null;
+    const n = Number(value);
+    return Number.isFinite(n) ? n : null;
+}
+const round = (n, digits = 2) => Math.round(n * 10 ** digits) / 10 ** digits;
+export function summariseLocust(files, maxFailRatio) {
+    const stats = toRecords(files.stats);
+    const total = stats.find((r) => r.Name === "Aggregated");
+    if (!total)
+        throw new Error('No "Aggregated" row in the stats CSV - is this a Locust *_stats.csv file?');
+    const requests = num(total["Request Count"]) ?? 0;
+    const failed = num(total["Failure Count"]) ?? 0;
+    const metrics = { requests, failures: failed };
+    const put = (key, value) => {
+        if (value !== null)
+            metrics[key] = round(value);
+    };
+    put("avg_ms", num(total["Average Response Time"]));
+    put("p50_ms", num(total["50%"]) ?? num(total["Median Response Time"]));
+    put("p95_ms", num(total["95%"]));
+    put("p99_ms", num(total["99%"]));
+    put("max_ms", num(total["Max Response Time"]));
+    put("rps", num(total["Requests/s"]));
+    const ratio = requests > 0 ? failed / requests : 0;
+    metrics.error_rate_pct = round(ratio * 100);
+    const exceptions = files.exceptions ? toRecords(files.exceptions).length : 0;
+    if (files.exceptions !== undefined)
+        metrics.exceptions = exceptions;
+    const failures = (files.failures ? toRecords(files.failures) : []).map((r) => ({
+        name: `${r.Method ?? ""} ${r.Name ?? ""}`.trim() || "(unnamed request)",
+        message: r.Error ? `${r.Error}${r.Occurrences ? ` (x${r.Occurrences})` : ""}` : null,
+    }));
+    let duration_ms = null;
+    if (files.history) {
+        const stamps = toRecords(files.history)
+            .map((r) => num(r.Timestamp))
+            .filter((n) => n !== null);
+        if (stamps.length > 1)
+            duration_ms = Math.round((Math.max(...stamps) - Math.min(...stamps)) * 1000);
+    }
+    const reasons = [];
+    if (exceptions > 0)
+        reasons.push(`${exceptions} unhandled task exception(s)`);
+    if (ratio > maxFailRatio) {
+        reasons.push(`failure ratio ${(ratio * 100).toFixed(1)}% > allowed ${(maxFailRatio * 100).toFixed(1)}%`);
+    }
+    return {
+        total: requests,
+        passed: requests - failed,
+        failed,
+        duration_ms,
+        metrics,
+        failures,
+        status: reasons.length === 0 ? "passed" : "failed",
+        reasons,
+    };
+}

package/dist/reports/specmatic.js

@@ -0,0 +1,69 @@
+/**
+ * Specmatic's API coverage -> the `coverage_pct` metric of a `contract` suite.
+ *
+ * `specmatic test` writes JUnit XML (the pass/fail counts) and an HTML report. The coverage - how many of
+ * the spec's operations a test actually exercised - is only in that report, as the JSON the page renders
+ * from (`const report = {...}`, a CTRF-style document: `results.summary.extra.executionDetails[]` with a
+ * `coverageMetrics` per spec). This reads the number out of either that HTML file or a plain JSON copy of
+ * the same document, so the spec-derived coverage reaches Assurance without the spec itself being uploaded.
+ *
+ * The HTML embedding is Specmatic's own, not a documented interface: when it changes the parse fails with
+ * a message naming the file, and the upload (without coverage) is still one flag away.
+ */
+/** The end index (exclusive) of the JSON object that starts at `start`, honouring strings and escapes. */
+function objectEnd(text, start) {
+    let depth = 0;
+    let inString = false;
+    for (let i = start; i < text.length; i++) {
+        const c = text[i];
+        if (inString) {
+            if (c === "\\")
+                i++;
+            else if (c === '"')
+                inString = false;
+        }
+        else if (c === '"')
+            inString = true;
+        else if (c === "{")
+            depth++;
+        else if (c === "}" && --depth === 0)
+            return i + 1;
+    }
+    throw new Error("unterminated JSON object");
+}
+function reportJson(text) {
+    const trimmed = text.trimStart();
+    if (trimmed.startsWith("{"))
+        return JSON.parse(trimmed);
+    const marker = "const report = ";
+    const at = text.indexOf(marker);
+    if (at < 0)
+        throw new Error("no embedded report found");
+    const start = at + marker.length;
+    return JSON.parse(text.slice(start, objectEnd(text, start)));
+}
+export function parseSpecmaticCoverage(text) {
+    let doc;
+    try {
+        doc = reportJson(text);
+    }
+    catch (err) {
+        throw new Error(`Cannot read the Specmatic report (${err instanceof Error ? err.message : String(err)})`);
+    }
+    const details = doc?.results?.summary?.extra?.executionDetails;
+    if (!Array.isArray(details) || details.length === 0) {
+        throw new Error("The Specmatic report has no executionDetails (was it written by `specmatic test`?)");
+    }
+    let covered = 0;
+    let total = 0;
+    for (const d of details) {
+        const m = d.coverageMetrics;
+        if (!m)
+            continue;
+        covered += m.coveredOperations ?? 0;
+        total += m.totalOperationsWithFilters ?? m.totalOperations ?? 0;
+    }
+    if (total <= 0)
+        throw new Error("The Specmatic report lists no operations to cover");
+    return { coverage_pct: Math.round((covered / total) * 1000) / 10, covered, total };
+}

package/dist/reports/upload.js

@@ -0,0 +1,335 @@
+/**
+ * `nb assurance report upload` - read a test report on the CI runner, reduce it
+ * to the normalised summary, and report it to a NASEBANAL Assurance project.
+ *
+ * The same command runs in a pipeline and on a laptop (nb-quickstarts' `make
+ * <tool>:report-upload` calls it): it takes its run key, commit, branch and
+ * project from the CI environment when there is one, and authenticates with
+ * `NB_TOKEN` (a PAT or service-account token) so no browser is needed.
+ *
+ * The parsing happens here, not on the server: the server stays small, parsers
+ * are testable offline, and what is uploaded is a few hundred bytes of summary
+ * instead of a megabyte of report.
+ */
+import { Command } from "commander";
+import { existsSync, readFileSync, readdirSync } from "node:fs";
+import { basename, dirname, join } from "node:path";
+import { resolveAccessToken, resolveEnv } from "../config.js";
+import { request, resolveBaseUrl, ApiError } from "../http.js";
+import { loadSpec } from "../spec/loader.js";
+import { printJson } from "../output.js";
+import { detectCi } from "./ci.js";
+import { parseJunit } from "./junit.js";
+import { summariseLocust } from "./locust.js";
+import { parseSpecmaticCoverage } from "./specmatic.js";
+export const SUITE_KINDS = ["unit", "e2e", "load", "contract", "security"];
+/**
+ * The tool is a label (nb-assurance-api `TOOL_PATTERN`): what a result means is carried by its kind and its
+ * numbers, so any scanner or runner can be reported without a change here, as long as its report is
+ * JUnit XML (or the numbers are given with --metric). Only the formats this CLI can read matter.
+ */
+const TOOL_PATTERN = /^[a-z0-9][a-z0-9._-]{0,49}$/;
+/** Tools whose report is not JUnit XML; every other tool is read as JUnit. */
+const NON_JUNIT_TOOLS = { locust: "locust" };
+/**
+ * Check a kind / tool / format combination before anything is parsed or sent.
+ * Returns the normalised (lower-case) tool label; throws a message naming what is wrong.
+ */
+export function checkSuite(kind, tool, format) {
+    const name = tool.trim().toLowerCase();
+    if (!TOOL_PATTERN.test(name)) {
+        throw new Error(`Tool '${tool}' is not a label: use lower-case letters, digits, '.', '_' and '-' (up to 50), e.g. vitest or trivy`);
+    }
+    const expected = NON_JUNIT_TOOLS[name] ?? "junit";
+    if (format !== expected) {
+        throw new Error(`Tool '${name}' reports in ${expected} format, but --format is ${format}`);
+    }
+    if (kind === "load" && name !== "locust" && format !== "junit")
+        throw new Error("Only locust reports are read as CSV");
+    return name;
+}
+/** The same label rule as the API (`utils/apiVersion.ts` in nb-assurance-api): checked here so a typo fails before anything is sent. */
+const API_VERSION_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._-]{0,31}$/;
+const METRIC_KEY = /^[a-z][a-z0-9_]{0,39}$/;
+const MAX_METRICS = 20;
+/** `--metric high=0 --metric medium=3`: extra numbers for the suite (the API keeps at most 20). */
+export function parseMetrics(pairs) {
+    const out = {};
+    for (const pair of pairs) {
+        const eq = pair.indexOf("=");
+        const key = eq < 0 ? pair : pair.slice(0, eq).trim();
+        const raw = eq < 0 ? "" : pair.slice(eq + 1).trim();
+        const value = Number(raw);
+        if (!METRIC_KEY.test(key) || raw === "" || !Number.isFinite(value)) {
+            throw new Error(`--metric '${pair}' must be <key>=<number> with a snake_case key (e.g. high=0)`);
+        }
+        out[key] = value;
+    }
+    if (Object.keys(out).length > MAX_METRICS)
+        throw new Error(`At most ${MAX_METRICS} --metric values per suite`);
+    return out;
+}
+export function checkApiVersion(label) {
+    const v = label.trim();
+    if (!API_VERSION_PATTERN.test(v)) {
+        throw new Error(`--api-version '${label}' is not a version label: use letters, digits, '.', '_' and '-' (up to 32), e.g. v1 or 2026-10`);
+    }
+    return v;
+}
+const MAX_FAILURES = 20;
+/** Per-test cases: the API accepts this many (MAX_CASES), each message up to 2000 characters. */
+export const MAX_CASES = 5000;
+const MAX_CASE_MESSAGE = 2000;
+/** Total failure text sent across all cases, so a suite where everything fails stays a modest upload. */
+const MAX_CASE_MESSAGE_BUDGET = 1_000_000;
+const clip = (s, max) => (s.length > max ? s.slice(0, max - 1) + "…" : s);
+function toFailures(list) {
+    const out = list.slice(0, MAX_FAILURES).map((f) => ({
+        name: clip(f.name, 300),
+        ...(f.message ? { message: clip(f.message, 500) } : {}),
+    }));
+    return out.length > 0 ? out : undefined;
+}
+/**
+ * The per-test cases to send. Over the limit, failed and skipped tests are kept first (they are
+ * what people open the list for) and passed ones fill the rest, all in their original order.
+ * Failure text is clipped per case and against a total budget; names and outcomes always stay.
+ */
+export function toCases(list) {
+    if (list.length === 0)
+        return undefined;
+    let keep = list.map((_, i) => i);
+    if (list.length > MAX_CASES) {
+        const notPassed = keep.filter((i) => list[i].status !== "passed");
+        const passed = keep.filter((i) => list[i].status === "passed");
+        keep = [...notPassed, ...passed].slice(0, MAX_CASES).sort((a, b) => a - b);
+    }
+    let budget = MAX_CASE_MESSAGE_BUDGET;
+    return keep.map((i) => {
+        const c = list[i];
+        let message;
+        if (c.message && budget > 0) {
+            message = clip(c.message, Math.min(MAX_CASE_MESSAGE, budget));
+            budget -= message.length;
+        }
+        return {
+            name: clip(c.name, 300),
+            status: c.status,
+            ...(c.duration_ms !== null ? { duration_ms: c.duration_ms } : {}),
+            ...(message ? { message } : {}),
+        };
+    });
+}
+/** The sibling file Locust wrote next to `<prefix>_stats.csv`, if there is one. */
+function locustSibling(statsFile, suffix) {
+    const path = statsFile.replace(/_stats\.csv$/, suffix);
+    return path !== statsFile && existsSync(path) ? readFileSync(path, "utf8") : undefined;
+}
+export function parseReport(opts) {
+    const text = readFileSync(opts.file, "utf8");
+    const base = { kind: opts.kind, tool: opts.tool, ...(opts.suiteName ? { name: opts.suiteName } : {}) };
+    if (opts.format === "junit") {
+        const j = parseJunit(text);
+        const status = opts.status ?? (j.failed === 0 ? "passed" : "failed");
+        return {
+            note: `${j.total} tests, ${j.failed} failed, ${j.skipped} skipped`,
+            suite: {
+                ...base,
+                status,
+                tests: { total: j.total, passed: j.passed, failed: j.failed, skipped: j.skipped },
+                ...(j.duration_ms !== null ? { duration_ms: j.duration_ms } : {}),
+                ...(toFailures(j.failures) ? { failures: toFailures(j.failures) } : {}),
+                ...(toCases(j.cases) ? { cases: toCases(j.cases) } : {}),
+            },
+        };
+    }
+    const l = summariseLocust({
+        stats: text,
+        failures: locustSibling(opts.file, "_failures.csv"),
+        exceptions: locustSibling(opts.file, "_exceptions.csv"),
+        history: locustSibling(opts.file, "_stats_history.csv"),
+    }, opts.maxFailRatio);
+    return {
+        note: `${l.total} requests, ${l.failed} failed${l.reasons.length ? ` (${l.reasons.join("; ")})` : ""}`,
+        suite: {
+            ...base,
+            status: opts.status ?? l.status,
+            tests: { total: l.total, passed: l.passed, failed: l.failed },
+            ...(l.duration_ms !== null ? { duration_ms: l.duration_ms } : {}),
+            metrics: l.metrics,
+            ...(toFailures(l.failures) ? { failures: toFailures(l.failures) } : {}),
+        },
+    };
+}
+/** Find the project by name, or create it (unless told not to). Returns its id. */
+export async function ensureProject(conn, name, repo, create) {
+    const find = async () => {
+        const res = await request({ ...conn, method: "GET", path: "/api/v1/projects", query: { name } });
+        const list = res.data.data.data;
+        return list[0]?.id ?? null;
+    };
+    const existing = await find();
+    if (existing)
+        return { id: existing, created: false };
+    if (!create) {
+        throw new Error(`No project named '${name}' (create it first, or drop --no-create-project).`);
+    }
+    try {
+        const res = await request({
+            ...conn,
+            method: "POST",
+            path: "/api/v1/projects",
+            body: { name, ...(repo ? { repo } : {}) },
+        });
+        return { id: res.data.data.id, created: true };
+    }
+    catch (err) {
+        // Another job of the same pipeline created it between our lookup and our create.
+        if (err instanceof ApiError && err.status === 409) {
+            const raced = await find();
+            if (raced)
+                return { id: raced, created: false };
+        }
+        throw err;
+    }
+}
+export async function uploadReport(p) {
+    const project = await ensureProject(p.conn, p.project, p.repo, p.createProject);
+    const res = await request({
+        ...p.conn,
+        method: "POST",
+        path: `/api/v1/projects/${encodeURIComponent(project.id)}/results`,
+        body: p.payload,
+    });
+    const data = res.data.data;
+    return { project_id: project.id, project_created: project.created, ...data };
+}
+function fail(message) {
+    throw new Error(message);
+}
+/** `--dir`: the newest-looking `*_stats.csv` (Locust writes `<prefix>_stats.csv` per run). */
+function statsFileIn(dir) {
+    const files = existsSync(dir) ? readdirSync(dir).filter((f) => f.endsWith("_stats.csv")) : [];
+    if (files.length === 0)
+        fail(`No *_stats.csv file in ${dir}`);
+    return join(dir, files.sort()[files.length - 1]);
+}
+function resolveFormat(opts, file) {
+    if (opts.format !== undefined) {
+        if (opts.format !== "junit" && opts.format !== "locust")
+            fail("--format must be junit or locust");
+        return opts.format;
+    }
+    if (opts.dir !== undefined || file.endsWith(".csv"))
+        return "locust";
+    if (file.endsWith(".xml"))
+        return "junit";
+    return fail("Cannot tell the report format from the file name; pass --format junit|locust");
+}
+export function buildReportUploadCommand() {
+    return new Command("upload")
+        .description("Report a test result file (JUnit XML or Locust CSV) to an Assurance project")
+        .option("--file <path>", "the report file (JUnit XML, or Locust *_stats.csv)")
+        .option("--dir <dir>", "a Locust output directory (uses its *_stats.csv)")
+        .option("--coverage-report <path>", "specmatic: its report (the HTML or a JSON copy); adds the API coverage as the coverage_pct metric")
+        .option("--format <format>", "junit | locust (default: from the file name)")
+        .option("--kind <kind>", `what the suite proves: ${SUITE_KINDS.join(" | ")} (locust: load)`)
+        .option("--tool <label>", "the producing tool, a free label, e.g. vitest, pytest, playwright, specmatic, zap, trivy (locust: locust)")
+        .option("--suite <name>", "tells apart several suites of one kind in a run (e.g. api, web)")
+        .option("--metric <key=value>", "an extra number for the suite, repeatable (e.g. --metric high=0 --metric medium=3; for security: critical, high, medium, low, info)", (v, all = []) => [...all, v])
+        .option("--api-version <label>", "the API version the result is for (e.g. v1, v2), part of the suite's identity next to its kind and name; leave it out for results that hold for every version, such as unit tests")
+        .option("--project <name>", "project name (CI: the repository); found, or created if missing")
+        .option("--repo <repo>", "repository recorded on a newly created project (CI: detected)")
+        .option("--run-key <key>", "identity of the pipeline run (CI: detected); all jobs of a run share it")
+        .option("--commit <sha>", "commit SHA (CI: detected)")
+        .option("--branch <name>", "branch (CI: detected)")
+        .option("--ci-url <url>", "link to the CI run (CI: detected)")
+        .option("--status <status>", "passed | failed; default is derived from the report")
+        .option("--max-fail-ratio <ratio>", "locust: allowed failure ratio 0..1 (default: $LOCUST_MAX_FAIL_RATIO or 0)")
+        .option("--no-create-project", "fail instead of creating the project when it does not exist")
+        .option("--dry-run", "print what would be sent and exit without calling the API")
+        .action(async (_flags, command) => {
+        const opts = command.optsWithGlobals();
+        const file = opts.file ?? (opts.dir !== undefined ? statsFileIn(opts.dir) : fail("Pass --file <path> (or --dir for Locust)"));
+        if (!existsSync(file))
+            fail(`File not found: ${file}`);
+        const format = resolveFormat(opts, file);
+        const kind = (opts.kind ?? (format === "locust" ? "load" : undefined));
+        if (!kind || !SUITE_KINDS.includes(kind))
+            fail(`--kind is required: ${SUITE_KINDS.join(" | ")}`);
+        const tool = opts.tool ?? (format === "locust" ? "locust" : undefined);
+        if (!tool)
+            fail("--tool is required for JUnit reports (e.g. --tool pytest)");
+        const checkedTool = checkSuite(kind, tool, format);
+        if (opts.status !== undefined && opts.status !== "passed" && opts.status !== "failed")
+            fail("--status must be passed or failed");
+        const ratioRaw = opts.maxFailRatio ?? process.env.LOCUST_MAX_FAIL_RATIO ?? "0";
+        const maxFailRatio = Number(ratioRaw);
+        if (!Number.isFinite(maxFailRatio) || maxFailRatio < 0 || maxFailRatio > 1)
+            fail("--max-fail-ratio must be a number between 0 and 1");
+        const apiVersion = opts.apiVersion === undefined ? undefined : checkApiVersion(opts.apiVersion);
+        const parsed = parseReport({
+            format,
+            kind,
+            tool: checkedTool,
+            suiteName: opts.suite,
+            status: opts.status,
+            maxFailRatio,
+            file,
+        });
+        if (opts.coverageReport !== undefined) {
+            if (kind !== "contract" || checkedTool !== "specmatic")
+                fail("--coverage-report is only for --kind contract --tool specmatic");
+            if (!existsSync(opts.coverageReport))
+                fail(`File not found: ${opts.coverageReport}`);
+            const cov = parseSpecmaticCoverage(readFileSync(opts.coverageReport, "utf8"));
+            parsed.suite.metrics = { ...parsed.suite.metrics, coverage_pct: cov.coverage_pct };
+            parsed.note += `, API coverage ${cov.coverage_pct}% (${cov.covered}/${cov.total} operations)`;
+        }
+        if (apiVersion !== undefined)
+            parsed.suite.api_version = apiVersion;
+        if (opts.metric !== undefined && opts.metric.length > 0) {
+            const extra = parseMetrics(opts.metric);
+            const merged = { ...parsed.suite.metrics, ...extra };
+            if (Object.keys(merged).length > MAX_METRICS)
+                fail(`At most ${MAX_METRICS} metrics per suite`);
+            parsed.suite.metrics = merged;
+            parsed.note += `, metrics ${Object.entries(extra).map(([k, v]) => `${k}=${v}`).join(" ")}`;
+        }
+        const ci = detectCi(process.env);
+        const runKey = opts.runKey ?? ci.runKey ?? fail("Cannot determine the run key; pass --run-key");
+        const project = opts.project ?? ci.project ?? fail("Cannot determine the project name; pass --project <name>");
+        const commit = opts.commit ?? ci.commitSha;
+        const branch = opts.branch ?? ci.branch;
+        const ciUrl = opts.ciUrl ?? ci.ciUrl;
+        const payload = {
+            run: {
+                key: runKey,
+                ...(commit ? { commit_sha: commit } : {}),
+                ...(branch ? { branch } : {}),
+                ...(ciUrl ? { ci_url: ciUrl } : {}),
+            },
+            suite: parsed.suite,
+        };
+        const repo = opts.repo ?? ci.repo;
+        if (opts.dryRun) {
+            printJson({ project, repo, payload });
+            process.stderr.write(`(dry run) ${basename(dirname(file)) || "."}/${basename(file)}: ${parsed.note}\n`);
+            return;
+        }
+        const env = opts.env || resolveEnv();
+        const override = process.env.NB_BASE_URL?.replace(/\/$/, "");
+        const baseUrl = override ?? resolveBaseUrl(loadSpec("assurance.yaml").servers, env);
+        const token = opts.token || (await resolveAccessToken());
+        const result = await uploadReport({
+            conn: { baseUrl, token },
+            project,
+            repo,
+            createProject: opts.createProject,
+            payload,
+        });
+        printJson(result);
+        process.stderr.write(`✓ ${kind}/${checkedTool} ${payload.suite.status}: ${parsed.note} -> project '${project}'` +
+            `${result.project_created ? " (created)" : ""}, run '${runKey}'${result.replaced ? " (replaced)" : ""}\n`);
+    });
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@nasebanal/cli",
-  "version": "0.1.0",
+  "version": "0.3.0",
   "description": "gcloud-style command-line interface for NASEBANAL APIs, generated from nb-api-specs OpenAPI contracts.",
   "license": "MIT",
   "type": "module",