@testsmith/api-spector 0.4.9 → 0.5.1

package/out/main/coverage.js

@@ -0,0 +1,161 @@
+#!/usr/bin/env node
+"use strict";
+const promises = require("fs/promises");
+const jsYaml = require("js-yaml");
+const undici = require("undici");
+const coverage = require("./chunks/coverage-C54mZEh_.js");
+const cliCommon = require("./chunks/cli-common-BKYgsbGB.js");
+require("path");
+require("crypto");
+async function loadObservations(path) {
+  const report = JSON.parse(await promises.readFile(path, "utf8"));
+  const out = [];
+  for (const r of report.results ?? []) {
+    if (!r.method || !r.url) continue;
+    const status = r.httpStatus ?? r.status;
+    if (typeof status !== "number") continue;
+    let responsePaths;
+    try {
+      if (r.response?.body) responsePaths = coverage.flattenValuePaths(JSON.parse(r.response.body));
+    } catch {
+    }
+    out.push({ method: r.method, url: r.url, status, responsePaths });
+  }
+  return out;
+}
+async function loadSpec(source) {
+  const isUrl = /^https?:\/\//i.test(source);
+  const raw = isUrl ? await (async () => {
+    const r = await undici.fetch(source);
+    if (!r.ok) throw new Error(`HTTP ${r.status} fetching ${source}`);
+    return r.text();
+  })() : await promises.readFile(source, "utf8");
+  const isYaml = /\.ya?ml$/i.test(source) || !source.trim().startsWith("{") && !isUrl;
+  return isYaml ? jsYaml.load(raw) : JSON.parse(raw);
+}
+function collectRequests(collections) {
+  const out = [];
+  for (const col of collections) {
+    for (const req of Object.values(col.requests ?? {})) {
+      if (req.disabled) continue;
+      out.push({
+        name: `${col.name} / ${req.name}`,
+        method: req.method,
+        url: req.url,
+        expectedStatus: req.contract?.statusCode
+      });
+    }
+  }
+  return out;
+}
+function bar(pct) {
+  const filled = Math.round(pct / 5);
+  return "█".repeat(filled) + "░".repeat(20 - filled);
+}
+function pctColor(pct) {
+  return pct >= 80 ? cliCommon.C.green : pct >= 50 ? cliCommon.C.yellow : cliCommon.C.red;
+}
+function printReport(report) {
+  const t = report.totals;
+  const title = report.spec.title ? `${report.spec.title}${report.spec.version ? ` v${report.spec.version}` : ""}` : "API";
+  console.log("");
+  console.log(cliCommon.color(`  ${title} - test coverage`, cliCommon.C.bold, cliCommon.C.white));
+  console.log("");
+  console.log(`  ${cliCommon.color(bar(t.operationPct), pctColor(t.operationPct))}  ${cliCommon.color(`${t.operationPct}%`, cliCommon.C.bold)} operations tested  (${t.tested}/${t.operations})`);
+  console.log(`  ${cliCommon.color(`${t.coveredStatuses}/${t.declaredStatuses}`, cliCommon.C.white)} declared response codes covered  (${t.statusPct}%)`);
+  if (t.declaredProperties > 0) {
+    console.log(`  ${cliCommon.color(`${t.coveredProperties}/${t.declaredProperties}`, cliCommon.C.white)} response-schema properties seen in runs  (${t.propertyPct}%)`);
+  }
+  console.log(`  ${cliCommon.color(String(t.untested), t.untested ? cliCommon.C.yellow : cliCommon.C.green)} operations never tested, ${cliCommon.color(String(t.withoutNegativeTest), t.withoutNegativeTest ? cliCommon.C.yellow : cliCommon.C.green)} without a negative test`);
+  console.log("");
+  for (const op of report.operations) {
+    const mark = op.tested ? cliCommon.color("✓", cliCommon.C.green) : cliCommon.color("✗", cliCommon.C.red);
+    const label = `${op.method.padEnd(6)} ${op.path}`;
+    const statuses = op.declaredStatuses.length ? cliCommon.color(`  [${op.coveredStatuses.length}/${op.declaredStatuses.length} codes]`, cliCommon.C.gray) : "";
+    const neg = op.tested && !op.hasNegativeTest ? cliCommon.color("  no negative test", cliCommon.C.yellow) : "";
+    console.log(`  ${mark} ${op.tested ? cliCommon.color(label, cliCommon.C.white) : cliCommon.color(label, cliCommon.C.dim)}${statuses}${neg}`);
+  }
+  console.log("");
+}
+function toHtml(report) {
+  const t = report.totals;
+  const rows = report.operations.map((op) => `
+    <tr class="${op.tested ? "ok" : "miss"}">
+      <td>${op.tested ? "✓" : "✗"}</td>
+      <td><code>${op.method}</code></td>
+      <td><code>${op.path}</code></td>
+      <td>${op.declaredStatuses.length ? `${op.coveredStatuses.length}/${op.declaredStatuses.length}` : "-"}</td>
+      <td>${op.tested ? op.hasNegativeTest ? "yes" : '<span class="warn">missing</span>' : "-"}</td>
+      <td>${op.requests.map((r) => r.replace(/</g, "&lt;")).join("<br>") || "-"}</td>
+    </tr>`).join("");
+  const title = (report.spec.title ?? "API") + (report.spec.version ? ` v${report.spec.version}` : "");
+  return `<!doctype html><meta charset="utf-8"><title>${title} coverage</title>
+<style>
+  body{font:14px/1.5 -apple-system,Segoe UI,Roboto,sans-serif;margin:2rem;color:#1c1b24}
+  h1{font-size:1.4rem} .big{font-size:2rem;font-weight:700}
+  .summary{display:flex;gap:2rem;margin:1rem 0 1.5rem}
+  table{border-collapse:collapse;width:100%} th,td{text-align:left;padding:.4rem .6rem;border-bottom:1px solid #eee;vertical-align:top}
+  code{background:#f4f4f5;padding:.1rem .3rem;border-radius:4px} tr.miss td{color:#b91c1c} .warn{color:#b45309}
+  th{color:#666;font-weight:600;font-size:.8rem;text-transform:uppercase;letter-spacing:.03em}
+</style>
+<h1>${title} - test coverage</h1>
+<div class="summary">
+  <div><div class="big">${t.operationPct}%</div>operations tested (${t.tested}/${t.operations})</div>
+  <div><div class="big">${t.statusPct}%</div>response codes (${t.coveredStatuses}/${t.declaredStatuses})</div>
+  <div><div class="big">${t.untested}</div>never tested</div>
+  <div><div class="big">${t.withoutNegativeTest}</div>without negative test</div>
+</div>
+<table><thead><tr><th></th><th>Method</th><th>Operation</th><th>Codes</th><th>Negative</th><th>Tests</th></tr></thead>
+<tbody>${rows}</tbody></table>`;
+}
+async function main() {
+  const args = cliCommon.parseArgs(process.argv.slice(2));
+  if (args.help || args.h) {
+    console.log("\nUsage:\n  api-spector coverage --workspace <path> --spec <file|url> [--json] [--output <file>] [--fail-under <pct>] [--collection <name>]\n");
+    process.exit(0);
+  }
+  const wsPath = args.workspace;
+  if (!wsPath) {
+    console.error("Error: --workspace is required.");
+    process.exit(2);
+  }
+  const { workspace, dir } = await cliCommon.loadWorkspace(wsPath);
+  const specSource = args.spec || workspace.settings?.coverageSpec;
+  if (!specSource) {
+    console.error("Error: --spec <file|url> is required (or set settings.coverageSpec in the workspace).");
+    process.exit(2);
+  }
+  let spec;
+  try {
+    spec = await loadSpec(specSource);
+  } catch (err) {
+    console.error(`Error loading spec: ${err instanceof Error ? err.message : String(err)}`);
+    process.exit(2);
+  }
+  const collections = await cliCommon.loadCollections(workspace, dir, { filterName: args.collection });
+  const requests = collectRequests(collections);
+  const observations = args.runs ? await loadObservations(args.runs) : [];
+  const report = coverage.computeCoverage(spec, requests, observations);
+  if (args.output) {
+    const path = args.output;
+    const content = /\.html?$/i.test(path) ? toHtml(report) : JSON.stringify(report, null, 2);
+    await promises.writeFile(path, content, "utf8");
+    console.error(`Wrote ${path}`);
+  }
+  if (args.json) {
+    console.log(JSON.stringify(report, null, 2));
+  } else if (!args.output) {
+    printReport(report);
+  }
+  if (args["fail-under"] !== void 0) {
+    const threshold = Number(args["fail-under"]);
+    if (!Number.isNaN(threshold) && report.totals.operationPct < threshold) {
+      console.error(`Coverage ${report.totals.operationPct}% is below --fail-under ${threshold}%.`);
+      process.exit(1);
+    }
+  }
+}
+main().catch((err) => {
+  console.error(err instanceof Error ? err.message : String(err));
+  process.exit(2);
+});

package/out/main/generate-tests.js

@@ -0,0 +1,317 @@
+#!/usr/bin/env node
+"use strict";
+const promises = require("fs/promises");
+const jsYaml = require("js-yaml");
+const undici = require("undici");
+const uuid = require("uuid");
+const coverage = require("./chunks/coverage-C54mZEh_.js");
+const cliCommon = require("./chunks/cli-common-BKYgsbGB.js");
+require("path");
+require("crypto");
+const HTTP_METHODS = ["get", "put", "post", "delete", "options", "head", "patch", "trace"];
+function resolveRef(spec, ref) {
+  const parts = ref.replace(/^#\//, "").split("/");
+  return parts.reduce((o, k) => o?.[decodeURIComponent(k.replace(/~1/g, "/").replace(/~0/g, "~"))], spec);
+}
+function deref(spec, node, depth = 0, seen = /* @__PURE__ */ new Set()) {
+  if (!node || typeof node !== "object" || depth > 8) return node;
+  if (Array.isArray(node)) return node.map((n) => deref(spec, n, depth + 1, seen));
+  if ("$ref" in node) {
+    const target = resolveRef(spec, node.$ref);
+    if (!target || seen.has(target)) return {};
+    return deref(spec, target, depth + 1, /* @__PURE__ */ new Set([...seen, target]));
+  }
+  const out = {};
+  for (const [k, v] of Object.entries(node)) out[k] = deref(spec, v, depth + 1, seen);
+  return out;
+}
+function jsonSchemaFor(spec, responses, code) {
+  const resp = responses?.[code];
+  const schema = resp?.content?.["application/json"]?.schema ?? resp?.content?.["application/json;charset=utf-8"]?.schema;
+  return schema ? deref(spec, schema) : void 0;
+}
+function sampleValue(schema) {
+  if (!schema || typeof schema !== "object") return "string";
+  if (schema.example !== void 0) return schema.example;
+  if (schema.default !== void 0) return schema.default;
+  if (Array.isArray(schema.enum) && schema.enum.length) return schema.enum[0];
+  const type = Array.isArray(schema.type) ? schema.type[0] : schema.type;
+  switch (type) {
+    case "integer":
+    case "number": {
+      const min = schema.minimum ?? (schema.exclusiveMinimum != null ? schema.exclusiveMinimum + 1 : void 0);
+      const max = schema.maximum ?? (schema.exclusiveMaximum != null ? schema.exclusiveMaximum - 1 : void 0);
+      if (min != null) return min;
+      if (max != null) return max;
+      return type === "integer" ? 1 : 1.5;
+    }
+    case "boolean":
+      return true;
+    case "array":
+      return [sampleValue(schema.items ?? {})];
+    case "object": {
+      const out = {};
+      const props = schema.properties ?? {};
+      const required = schema.required ?? [];
+      for (const [name, propSchema] of Object.entries(props)) out[name] = sampleValue(propSchema);
+      for (const name of required) if (!(name in out)) out[name] = "string";
+      return out;
+    }
+    case "string":
+    default:
+      return sampleString(schema);
+  }
+}
+function sampleString(schema) {
+  switch (schema.format) {
+    case "email":
+      return "user@example.com";
+    case "uuid":
+      return "00000000-0000-0000-0000-000000000000";
+    case "date":
+      return "2020-01-01";
+    case "date-time":
+      return "2020-01-01T00:00:00Z";
+    case "uri":
+    case "url":
+      return "https://example.com";
+    case "hostname":
+      return "example.com";
+    case "ipv4":
+      return "127.0.0.1";
+    default: {
+      const min = schema.minLength ?? 0;
+      let s = "string";
+      if (min > s.length) s = s.padEnd(min, "x");
+      if (schema.maxLength != null && s.length > schema.maxLength) s = s.slice(0, schema.maxLength);
+      return s;
+    }
+  }
+}
+function collectParams(spec, pathItem, op) {
+  const raw = [...pathItem?.parameters ?? [], ...op?.parameters ?? []].map((p) => deref(spec, p));
+  const byKey = /* @__PURE__ */ new Map();
+  for (const p of raw) if (p?.name && p?.in) byKey.set(`${p.in}:${p.name}`, p);
+  return [...byKey.values()];
+}
+function successCode(responses) {
+  const codes = Object.keys(responses ?? {}).filter((c) => /^2\d\d$/.test(c)).map(Number).sort((a, b) => a - b);
+  return codes[0] ?? 200;
+}
+function negativeCode(responses) {
+  const codes = Object.keys(responses ?? {}).filter((c) => /^4\d\d$/.test(c)).map(Number).sort((a, b) => a - b);
+  return codes[0] ?? 400;
+}
+function baseTest(method, path, op, params) {
+  const pathParams = {};
+  for (const p of params.filter((p2) => p2.in === "path")) pathParams[p.name] = sampleValue(p.schema ?? {});
+  const query = params.filter((p) => p.in === "query" && p.required).map((p) => ({ key: p.name, value: String(sampleValue(p.schema ?? {})) }));
+  const headers = params.filter((p) => p.in === "header" && p.required).map((p) => ({ key: p.name, value: String(sampleValue(p.schema ?? {})) }));
+  return { operationId: op?.operationId, method, path, pathParams, query, headers };
+}
+function requestBodySchema(spec, op) {
+  const rb = deref(spec, op?.requestBody);
+  const schema = rb?.content?.["application/json"]?.schema;
+  return schema;
+}
+function generateForOperation(spec, method, path, pathItem, op, opts) {
+  const tests = [];
+  const params = collectParams(spec, pathItem, op);
+  const responses = op?.responses ?? {};
+  const okCode = successCode(responses);
+  const badCode = negativeCode(responses);
+  const bodySchema = requestBodySchema(spec, op);
+  const validBody = bodySchema ? sampleValue(bodySchema) : void 0;
+  const base = baseTest(method, path, op, params);
+  const label = `${method} ${path}`;
+  tests.push({
+    ...base,
+    name: `${label} - happy path`,
+    category: "happy",
+    body: validBody !== void 0 ? JSON.stringify(validBody, null, 2) : void 0,
+    expectedStatus: okCode,
+    responseSchema: jsonSchemaFor(spec, responses, String(okCode)) ? JSON.stringify(jsonSchemaFor(spec, responses, String(okCode)), null, 2) : void 0
+  });
+  const props = bodySchema?.properties ?? {};
+  const required = bodySchema?.required ?? [];
+  if (opts.includeNegative && validBody && typeof validBody === "object") {
+    let n = 0;
+    for (const field of required) {
+      if (n >= opts.maxNegativePerOp) break;
+      const mutated = { ...validBody };
+      delete mutated[field];
+      tests.push({ ...base, name: `${label} - missing ${field}`, category: "negative", body: JSON.stringify(mutated, null, 2), expectedStatus: badCode });
+      n++;
+    }
+    for (const [field, ps] of Object.entries(props)) {
+      if (n >= opts.maxNegativePerOp) break;
+      const t = Array.isArray(ps.type) ? ps.type[0] : ps.type;
+      if (t !== "string" && t !== "integer" && t !== "number" && t !== "boolean") continue;
+      const wrong = t === "string" ? 12345 : "not-a-valid-value";
+      tests.push({ ...base, name: `${label} - ${field} wrong type`, category: "negative", body: JSON.stringify({ ...validBody, [field]: wrong }, null, 2), expectedStatus: badCode });
+      n++;
+    }
+  }
+  if (opts.includeBoundary && validBody && typeof validBody === "object") {
+    let n = 0;
+    for (const [field, ps] of Object.entries(props)) {
+      if (n >= opts.maxBoundaryPerOp) break;
+      const t = Array.isArray(ps.type) ? ps.type[0] : ps.type;
+      if ((t === "integer" || t === "number") && ps.minimum != null) {
+        tests.push({ ...base, name: `${label} - ${field} below minimum`, category: "boundary", body: JSON.stringify({ ...validBody, [field]: ps.minimum - 1 }, null, 2), expectedStatus: badCode });
+        n++;
+      } else if ((t === "integer" || t === "number") && ps.maximum != null) {
+        tests.push({ ...base, name: `${label} - ${field} above maximum`, category: "boundary", body: JSON.stringify({ ...validBody, [field]: ps.maximum + 1 }, null, 2), expectedStatus: badCode });
+        n++;
+      } else if (t === "string" && ps.maxLength != null) {
+        tests.push({ ...base, name: `${label} - ${field} too long`, category: "boundary", body: JSON.stringify({ ...validBody, [field]: "x".repeat(ps.maxLength + 1) }, null, 2), expectedStatus: badCode });
+        n++;
+      }
+    }
+  }
+  return tests;
+}
+function generateTests(spec, options = {}) {
+  const opts = {
+    only: options.only ?? /* @__PURE__ */ new Set(),
+    includeNegative: options.includeNegative ?? true,
+    includeBoundary: options.includeBoundary ?? true,
+    maxNegativePerOp: options.maxNegativePerOp ?? 4,
+    maxBoundaryPerOp: options.maxBoundaryPerOp ?? 4
+  };
+  const doc = spec ?? {};
+  const out = [];
+  for (const [path, item] of Object.entries(doc.paths ?? {})) {
+    if (!item || typeof item !== "object") continue;
+    for (const [method, op] of Object.entries(item)) {
+      if (!HTTP_METHODS.includes(method.toLowerCase())) continue;
+      if (!op || typeof op !== "object") continue;
+      if (opts.only.size && !opts.only.has(`${method.toUpperCase()} ${path}`)) continue;
+      out.push(...generateForOperation(doc, method.toUpperCase(), path, item, op, opts));
+    }
+  }
+  return out;
+}
+function testUrl(test, baseVar = "{{baseUrl}}") {
+  let path = test.path;
+  for (const [name, value] of Object.entries(test.pathParams)) {
+    path = path.replace(new RegExp(`\\{${name}\\}`, "g"), encodeURIComponent(String(value)));
+  }
+  const qs = test.query.filter((q) => q.key).map((q) => `${encodeURIComponent(q.key)}=${encodeURIComponent(q.value)}`).join("&");
+  return `${baseVar}${path}${qs ? `?${qs}` : ""}`;
+}
+function toApiRequest(test, id) {
+  const headers = test.headers.map((h) => ({ key: h.key, value: h.value, enabled: true }));
+  return {
+    id,
+    name: test.name,
+    method: test.method,
+    url: testUrl(test),
+    headers,
+    params: [],
+    auth: { type: "none" },
+    body: test.body !== void 0 ? { mode: "json", json: test.body } : { mode: "none" },
+    contract: {
+      statusCode: test.expectedStatus,
+      ...test.responseSchema ? { bodySchema: test.responseSchema } : {}
+    },
+    meta: { tags: [test.category] }
+  };
+}
+async function loadSpec(source) {
+  const isUrl = /^https?:\/\//i.test(source);
+  const raw = isUrl ? await (async () => {
+    const r = await undici.fetch(source);
+    if (!r.ok) throw new Error(`HTTP ${r.status} fetching ${source}`);
+    return r.text();
+  })() : await promises.readFile(source, "utf8");
+  return /\.ya?ml$/i.test(source) || !isUrl && !raw.trim().startsWith("{") ? jsYaml.load(raw) : JSON.parse(raw);
+}
+async function untestedOperations(specSource, spec, wsPath) {
+  const { workspace, dir } = await cliCommon.loadWorkspace(wsPath);
+  const collections = await cliCommon.loadCollections(workspace, dir);
+  const requests = [];
+  for (const col of collections) {
+    for (const req of Object.values(col.requests ?? {})) {
+      if (req.disabled) continue;
+      requests.push({ name: `${col.name} / ${req.name}`, method: req.method, url: req.url, expectedStatus: req.contract?.statusCode });
+    }
+  }
+  const report = coverage.computeCoverage(spec, requests);
+  return new Set(report.operations.filter((o) => !o.tested).map((o) => `${o.method} ${o.path}`));
+}
+function buildCollection(name, tests) {
+  const requests = {};
+  const requestIds = [];
+  for (const t of tests) {
+    const id = uuid.v4();
+    requests[id] = toApiRequest(t, id);
+    requestIds.push(id);
+  }
+  return {
+    version: "1.0",
+    id: uuid.v4(),
+    name,
+    description: "Generated from an OpenAPI spec by api-spector generate-tests.",
+    rootFolder: { id: uuid.v4(), name: "root", description: "", folders: [], requestIds },
+    requests
+  };
+}
+async function main() {
+  const args = cliCommon.parseArgs(process.argv.slice(2));
+  if (args.help || args.h) {
+    console.log("\nUsage:\n  api-spector generate-tests --spec <file|url> --output <collection.json> [--untested-only --workspace <ws>] [--name <label>] [--no-negative] [--no-boundary]\n");
+    process.exit(0);
+  }
+  const specSource = args.spec;
+  const output = args.output;
+  if (!specSource) {
+    console.error("Error: --spec <file|url> is required.");
+    process.exit(2);
+  }
+  if (!output) {
+    console.error("Error: --output <collection.json> is required.");
+    process.exit(2);
+  }
+  let spec;
+  try {
+    spec = await loadSpec(specSource);
+  } catch (err) {
+    console.error(`Error loading spec: ${err instanceof Error ? err.message : String(err)}`);
+    process.exit(2);
+  }
+  let only;
+  if (args["untested-only"]) {
+    if (!args.workspace) {
+      console.error("Error: --untested-only requires --workspace.");
+      process.exit(2);
+    }
+    only = await untestedOperations(specSource, spec, args.workspace);
+    if (only.size === 0) {
+      console.log(cliCommon.color("\n  Every operation already has a test. Nothing to generate.\n", cliCommon.C.green));
+      process.exit(0);
+    }
+  }
+  const tests = generateTests(spec, {
+    only,
+    includeNegative: args["no-negative"] !== true,
+    includeBoundary: args["no-boundary"] !== true
+  });
+  const specTitle = spec?.info?.title;
+  const name = args.name || `${specTitle ?? "API"} tests`;
+  const collection = buildCollection(name, tests);
+  await promises.writeFile(output, JSON.stringify(collection, null, 2), "utf8");
+  const byCat = tests.reduce((m, t) => {
+    m[t.category] = (m[t.category] ?? 0) + 1;
+    return m;
+  }, {});
+  console.log("");
+  console.log(cliCommon.color(`  Generated ${tests.length} tests`, cliCommon.C.bold, cliCommon.C.white) + cliCommon.color(` -> ${output}`, cliCommon.C.gray));
+  console.log(`  ${cliCommon.color(String(byCat.happy ?? 0), cliCommon.C.green)} happy path, ${cliCommon.color(String(byCat.negative ?? 0), cliCommon.C.yellow)} negative, ${cliCommon.color(String(byCat.boundary ?? 0), cliCommon.C.yellow)} boundary`);
+  console.log(cliCommon.color(`  Add it to a workspace, or open the app and import the collection.`, cliCommon.C.gray));
+  console.log("");
+}
+main().catch((err) => {
+  console.error(err instanceof Error ? err.message : String(err));
+  process.exit(2);
+});