@nasebanal/cli 0.1.0 → 0.2.0

package/dist/reports/locust.js

@@ -0,0 +1,68 @@
+/**
+ * Locust's CSV output (`--csv <prefix>`) -> the normalised `load` summary.
+ *
+ * The pass/fail rule is the one nb-quickstarts' `locust/bin/exit_code.py`
+ * applies to Locust's own exit code, reproduced from the files so a report can
+ * be uploaded after the run without carrying the exit code along: any unhandled
+ * task exception, or a failure ratio above the allowed one, fails the run.
+ */
+import { toRecords } from "./csv.js";
+function num(value) {
+    if (value === undefined || value === "" || value === "N/A")
+        return null;
+    const n = Number(value);
+    return Number.isFinite(n) ? n : null;
+}
+const round = (n, digits = 2) => Math.round(n * 10 ** digits) / 10 ** digits;
+export function summariseLocust(files, maxFailRatio) {
+    const stats = toRecords(files.stats);
+    const total = stats.find((r) => r.Name === "Aggregated");
+    if (!total)
+        throw new Error('No "Aggregated" row in the stats CSV - is this a Locust *_stats.csv file?');
+    const requests = num(total["Request Count"]) ?? 0;
+    const failed = num(total["Failure Count"]) ?? 0;
+    const metrics = { requests, failures: failed };
+    const put = (key, value) => {
+        if (value !== null)
+            metrics[key] = round(value);
+    };
+    put("avg_ms", num(total["Average Response Time"]));
+    put("p50_ms", num(total["50%"]) ?? num(total["Median Response Time"]));
+    put("p95_ms", num(total["95%"]));
+    put("p99_ms", num(total["99%"]));
+    put("max_ms", num(total["Max Response Time"]));
+    put("rps", num(total["Requests/s"]));
+    const ratio = requests > 0 ? failed / requests : 0;
+    metrics.error_rate_pct = round(ratio * 100);
+    const exceptions = files.exceptions ? toRecords(files.exceptions).length : 0;
+    if (files.exceptions !== undefined)
+        metrics.exceptions = exceptions;
+    const failures = (files.failures ? toRecords(files.failures) : []).map((r) => ({
+        name: `${r.Method ?? ""} ${r.Name ?? ""}`.trim() || "(unnamed request)",
+        message: r.Error ? `${r.Error}${r.Occurrences ? ` (x${r.Occurrences})` : ""}` : null,
+    }));
+    let duration_ms = null;
+    if (files.history) {
+        const stamps = toRecords(files.history)
+            .map((r) => num(r.Timestamp))
+            .filter((n) => n !== null);
+        if (stamps.length > 1)
+            duration_ms = Math.round((Math.max(...stamps) - Math.min(...stamps)) * 1000);
+    }
+    const reasons = [];
+    if (exceptions > 0)
+        reasons.push(`${exceptions} unhandled task exception(s)`);
+    if (ratio > maxFailRatio) {
+        reasons.push(`failure ratio ${(ratio * 100).toFixed(1)}% > allowed ${(maxFailRatio * 100).toFixed(1)}%`);
+    }
+    return {
+        total: requests,
+        passed: requests - failed,
+        failed,
+        duration_ms,
+        metrics,
+        failures,
+        status: reasons.length === 0 ? "passed" : "failed",
+        reasons,
+    };
+}

package/dist/reports/upload.js

@@ -0,0 +1,288 @@
+/**
+ * `nb assurance report upload` - read a test report on the CI runner, reduce it
+ * to the normalised summary, and report it to a NASEBANAL Assurance project.
+ *
+ * The same command runs in a pipeline and on a laptop (nb-quickstarts' `make
+ * <tool>:report-upload` calls it): it takes its run key, commit, branch and
+ * project from the CI environment when there is one, and authenticates with
+ * `NB_TOKEN` (a PAT or service-account token) so no browser is needed.
+ *
+ * The parsing happens here, not on the server: the server stays small, parsers
+ * are testable offline, and what is uploaded is a few hundred bytes of summary
+ * instead of a megabyte of report.
+ */
+import { Command } from "commander";
+import { existsSync, readFileSync, readdirSync } from "node:fs";
+import { basename, dirname, join } from "node:path";
+import { resolveAccessToken, resolveEnv } from "../config.js";
+import { request, resolveBaseUrl, ApiError } from "../http.js";
+import { loadSpec } from "../spec/loader.js";
+import { printJson } from "../output.js";
+import { detectCi } from "./ci.js";
+import { parseJunit } from "./junit.js";
+import { summariseLocust } from "./locust.js";
+export const SUITE_KINDS = ["unit", "e2e", "load", "contract", "security"];
+/**
+ * The tools Assurance accepts, per kind of suite. The CLI filters here, on the CI
+ * runner, so only reports that match are ever sent; the API enforces the same list
+ * (nb-assurance-api `ALLOWED_TOOLS`) because the CLI is not the only possible client.
+ * Keep the two in step with the `tool` enum in nb-api-specs' assurance package.
+ */
+export const ALLOWED_TOOLS = {
+    unit: ["vitest", "pytest"],
+    e2e: ["playwright"],
+    contract: ["specmatic"],
+    security: ["zap"],
+    load: ["locust"],
+};
+/** Tools whose report is not JUnit XML. Every other allowed tool writes JUnit. */
+const NON_JUNIT_TOOLS = { locust: "locust" };
+/**
+ * Check a kind / tool / format combination before anything is parsed or sent.
+ * Returns the normalised (lower-case) tool name; throws a message naming what is supported.
+ */
+export function checkSuite(kind, tool, format) {
+    const name = tool.trim().toLowerCase();
+    if (!ALLOWED_TOOLS[kind].includes(name)) {
+        throw new Error(`Tool '${tool}' is not supported for kind '${kind}'. Supported: ${ALLOWED_TOOLS[kind].join(", ")}`);
+    }
+    const expected = NON_JUNIT_TOOLS[name] ?? "junit";
+    if (format !== expected) {
+        throw new Error(`Tool '${name}' reports in ${expected} format, but --format is ${format}`);
+    }
+    return name;
+}
+const MAX_FAILURES = 20;
+/** Per-test cases: the API accepts this many (MAX_CASES), each message up to 2000 characters. */
+export const MAX_CASES = 5000;
+const MAX_CASE_MESSAGE = 2000;
+/** Total failure text sent across all cases, so a suite where everything fails stays a modest upload. */
+const MAX_CASE_MESSAGE_BUDGET = 1_000_000;
+const clip = (s, max) => (s.length > max ? s.slice(0, max - 1) + "…" : s);
+function toFailures(list) {
+    const out = list.slice(0, MAX_FAILURES).map((f) => ({
+        name: clip(f.name, 300),
+        ...(f.message ? { message: clip(f.message, 500) } : {}),
+    }));
+    return out.length > 0 ? out : undefined;
+}
+/**
+ * The per-test cases to send. Over the limit, failed and skipped tests are kept first (they are
+ * what people open the list for) and passed ones fill the rest, all in their original order.
+ * Failure text is clipped per case and against a total budget; names and outcomes always stay.
+ */
+export function toCases(list) {
+    if (list.length === 0)
+        return undefined;
+    let keep = list.map((_, i) => i);
+    if (list.length > MAX_CASES) {
+        const notPassed = keep.filter((i) => list[i].status !== "passed");
+        const passed = keep.filter((i) => list[i].status === "passed");
+        keep = [...notPassed, ...passed].slice(0, MAX_CASES).sort((a, b) => a - b);
+    }
+    let budget = MAX_CASE_MESSAGE_BUDGET;
+    return keep.map((i) => {
+        const c = list[i];
+        let message;
+        if (c.message && budget > 0) {
+            message = clip(c.message, Math.min(MAX_CASE_MESSAGE, budget));
+            budget -= message.length;
+        }
+        return {
+            name: clip(c.name, 300),
+            status: c.status,
+            ...(c.duration_ms !== null ? { duration_ms: c.duration_ms } : {}),
+            ...(message ? { message } : {}),
+        };
+    });
+}
+/** The sibling file Locust wrote next to `<prefix>_stats.csv`, if there is one. */
+function locustSibling(statsFile, suffix) {
+    const path = statsFile.replace(/_stats\.csv$/, suffix);
+    return path !== statsFile && existsSync(path) ? readFileSync(path, "utf8") : undefined;
+}
+export function parseReport(opts) {
+    const text = readFileSync(opts.file, "utf8");
+    const base = { kind: opts.kind, tool: opts.tool, ...(opts.suiteName ? { name: opts.suiteName } : {}) };
+    if (opts.format === "junit") {
+        const j = parseJunit(text);
+        const status = opts.status ?? (j.failed === 0 ? "passed" : "failed");
+        return {
+            note: `${j.total} tests, ${j.failed} failed, ${j.skipped} skipped`,
+            suite: {
+                ...base,
+                status,
+                tests: { total: j.total, passed: j.passed, failed: j.failed, skipped: j.skipped },
+                ...(j.duration_ms !== null ? { duration_ms: j.duration_ms } : {}),
+                ...(toFailures(j.failures) ? { failures: toFailures(j.failures) } : {}),
+                ...(toCases(j.cases) ? { cases: toCases(j.cases) } : {}),
+            },
+        };
+    }
+    const l = summariseLocust({
+        stats: text,
+        failures: locustSibling(opts.file, "_failures.csv"),
+        exceptions: locustSibling(opts.file, "_exceptions.csv"),
+        history: locustSibling(opts.file, "_stats_history.csv"),
+    }, opts.maxFailRatio);
+    return {
+        note: `${l.total} requests, ${l.failed} failed${l.reasons.length ? ` (${l.reasons.join("; ")})` : ""}`,
+        suite: {
+            ...base,
+            status: opts.status ?? l.status,
+            tests: { total: l.total, passed: l.passed, failed: l.failed },
+            ...(l.duration_ms !== null ? { duration_ms: l.duration_ms } : {}),
+            metrics: l.metrics,
+            ...(toFailures(l.failures) ? { failures: toFailures(l.failures) } : {}),
+        },
+    };
+}
+/** Find the project by name, or create it (unless told not to). Returns its id. */
+export async function ensureProject(conn, name, repo, create) {
+    const find = async () => {
+        const res = await request({ ...conn, method: "GET", path: "/api/v1/projects", query: { name } });
+        const list = res.data.data.data;
+        return list[0]?.id ?? null;
+    };
+    const existing = await find();
+    if (existing)
+        return { id: existing, created: false };
+    if (!create) {
+        throw new Error(`No project named '${name}' (create it first, or drop --no-create-project).`);
+    }
+    try {
+        const res = await request({
+            ...conn,
+            method: "POST",
+            path: "/api/v1/projects",
+            body: { name, ...(repo ? { repo } : {}) },
+        });
+        return { id: res.data.data.id, created: true };
+    }
+    catch (err) {
+        // Another job of the same pipeline created it between our lookup and our create.
+        if (err instanceof ApiError && err.status === 409) {
+            const raced = await find();
+            if (raced)
+                return { id: raced, created: false };
+        }
+        throw err;
+    }
+}
+export async function uploadReport(p) {
+    const project = await ensureProject(p.conn, p.project, p.repo, p.createProject);
+    const res = await request({
+        ...p.conn,
+        method: "POST",
+        path: `/api/v1/projects/${encodeURIComponent(project.id)}/results`,
+        body: p.payload,
+    });
+    const data = res.data.data;
+    return { project_id: project.id, project_created: project.created, ...data };
+}
+function fail(message) {
+    throw new Error(message);
+}
+/** `--dir`: the newest-looking `*_stats.csv` (Locust writes `<prefix>_stats.csv` per run). */
+function statsFileIn(dir) {
+    const files = existsSync(dir) ? readdirSync(dir).filter((f) => f.endsWith("_stats.csv")) : [];
+    if (files.length === 0)
+        fail(`No *_stats.csv file in ${dir}`);
+    return join(dir, files.sort()[files.length - 1]);
+}
+function resolveFormat(opts, file) {
+    if (opts.format !== undefined) {
+        if (opts.format !== "junit" && opts.format !== "locust")
+            fail("--format must be junit or locust");
+        return opts.format;
+    }
+    if (opts.dir !== undefined || file.endsWith(".csv"))
+        return "locust";
+    if (file.endsWith(".xml"))
+        return "junit";
+    return fail("Cannot tell the report format from the file name; pass --format junit|locust");
+}
+export function buildReportUploadCommand() {
+    return new Command("upload")
+        .description("Report a test result file (JUnit XML or Locust CSV) to an Assurance project")
+        .option("--file <path>", "the report file (JUnit XML, or Locust *_stats.csv)")
+        .option("--dir <dir>", "a Locust output directory (uses its *_stats.csv)")
+        .option("--format <format>", "junit | locust (default: from the file name)")
+        .option("--kind <kind>", `what the suite proves: ${SUITE_KINDS.join(" | ")} (locust: load)`)
+        .option("--tool <name>", "the producing tool, e.g. vitest, pytest, playwright, specmatic (locust: locust)")
+        .option("--suite <name>", "tells apart several suites of one kind in a run (e.g. api, web)")
+        .option("--project <name>", "project name (CI: the repository); found, or created if missing")
+        .option("--repo <repo>", "repository recorded on a newly created project (CI: detected)")
+        .option("--run-key <key>", "identity of the pipeline run (CI: detected); all jobs of a run share it")
+        .option("--commit <sha>", "commit SHA (CI: detected)")
+        .option("--branch <name>", "branch (CI: detected)")
+        .option("--ci-url <url>", "link to the CI run (CI: detected)")
+        .option("--status <status>", "passed | failed; default is derived from the report")
+        .option("--max-fail-ratio <ratio>", "locust: allowed failure ratio 0..1 (default: $LOCUST_MAX_FAIL_RATIO or 0)")
+        .option("--no-create-project", "fail instead of creating the project when it does not exist")
+        .option("--dry-run", "print what would be sent and exit without calling the API")
+        .action(async (_flags, command) => {
+        const opts = command.optsWithGlobals();
+        const file = opts.file ?? (opts.dir !== undefined ? statsFileIn(opts.dir) : fail("Pass --file <path> (or --dir for Locust)"));
+        if (!existsSync(file))
+            fail(`File not found: ${file}`);
+        const format = resolveFormat(opts, file);
+        const kind = (opts.kind ?? (format === "locust" ? "load" : undefined));
+        if (!kind || !SUITE_KINDS.includes(kind))
+            fail(`--kind is required: ${SUITE_KINDS.join(" | ")}`);
+        const tool = opts.tool ?? (format === "locust" ? "locust" : undefined);
+        if (!tool)
+            fail("--tool is required for JUnit reports (e.g. --tool pytest)");
+        const checkedTool = checkSuite(kind, tool, format);
+        if (opts.status !== undefined && opts.status !== "passed" && opts.status !== "failed")
+            fail("--status must be passed or failed");
+        const ratioRaw = opts.maxFailRatio ?? process.env.LOCUST_MAX_FAIL_RATIO ?? "0";
+        const maxFailRatio = Number(ratioRaw);
+        if (!Number.isFinite(maxFailRatio) || maxFailRatio < 0 || maxFailRatio > 1)
+            fail("--max-fail-ratio must be a number between 0 and 1");
+        const parsed = parseReport({
+            format,
+            kind,
+            tool: checkedTool,
+            suiteName: opts.suite,
+            status: opts.status,
+            maxFailRatio,
+            file,
+        });
+        const ci = detectCi(process.env);
+        const runKey = opts.runKey ?? ci.runKey ?? fail("Cannot determine the run key; pass --run-key");
+        const project = opts.project ?? ci.project ?? fail("Cannot determine the project name; pass --project <name>");
+        const commit = opts.commit ?? ci.commitSha;
+        const branch = opts.branch ?? ci.branch;
+        const ciUrl = opts.ciUrl ?? ci.ciUrl;
+        const payload = {
+            run: {
+                key: runKey,
+                ...(commit ? { commit_sha: commit } : {}),
+                ...(branch ? { branch } : {}),
+                ...(ciUrl ? { ci_url: ciUrl } : {}),
+            },
+            suite: parsed.suite,
+        };
+        const repo = opts.repo ?? ci.repo;
+        if (opts.dryRun) {
+            printJson({ project, repo, payload });
+            process.stderr.write(`(dry run) ${basename(dirname(file)) || "."}/${basename(file)}: ${parsed.note}\n`);
+            return;
+        }
+        const env = opts.env || resolveEnv();
+        const override = process.env.NB_BASE_URL?.replace(/\/$/, "");
+        const baseUrl = override ?? resolveBaseUrl(loadSpec("assurance.yaml").servers, env);
+        const token = opts.token || (await resolveAccessToken());
+        const result = await uploadReport({
+            conn: { baseUrl, token },
+            project,
+            repo,
+            createProject: opts.createProject,
+            payload,
+        });
+        printJson(result);
+        process.stderr.write(`✓ ${kind}/${checkedTool} ${payload.suite.status}: ${parsed.note} -> project '${project}'` +
+            `${result.project_created ? " (created)" : ""}, run '${runKey}'${result.replaced ? " (replaced)" : ""}\n`);
+    });
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@nasebanal/cli",
-  "version": "0.1.0",
+  "version": "0.2.0",
   "description": "gcloud-style command-line interface for NASEBANAL APIs, generated from nb-api-specs OpenAPI contracts.",
   "license": "MIT",
   "type": "module",