@raishin/vanguard-frontier-agentic 3.12.0 → 3.13.0

package/tests/integration/rbac-pre-flight/self-test.sh

@@ -0,0 +1,181 @@
+#!/usr/bin/env bash
+# tests/integration/rbac-pre-flight/self-test.sh
+#
+# Tests the harness itself, with no cluster.
+#
+# The guards assert things about RBAC manifests; nothing asserted anything
+# about the guards.  That gap shipped a fail-open: `_run_can_i` captured
+# `kubectl auth can-i` with `2>&1`, so any warning on stderr arrived glued to
+# the front of the verdict on stdout, matched neither `yes*` nor `no*`, and was
+# filed as a kubectl error.  In CI that turned 176 of 294 assertions into skips
+# while the log claimed the CRDs merely were not installed.
+#
+# Every expectation below is grounded in kubernetes/kubernetes
+# staging/src/k8s.io/kubectl/pkg/cmd/auth/cani.go, which is identical at
+# v1.28.15 and v1.31.2 (both ends of this suite's CI matrix):
+#
+#   * the verdict is written to o.Out   (RunAccessCheck)
+#   * every warning to o.ErrOut         (Complete, via WarningPrinter)
+#   * resourceFor() warns about an unresolvable resource and then returns
+#     GroupVersionResource{Resource: resourceArg} -- the full dotted argument
+#     as the resource, with an empty group -- so kubectl answers "no" to a
+#     question no RBAC rule can match.
+#
+# A stub `kubectl` on PATH replays those exact shapes.  Deterministic: no
+# cluster, no network, no clock.
+#
+# Usage: ./self-test.sh        (exit 0 = harness behaves correctly)
+
+set -uo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+STUB_DIR="$(mktemp -d "${TMPDIR:-/tmp}/rbac-selftest.XXXXXX")"
+trap 'rm -rf "$STUB_DIR"' EXIT
+
+# ---------------------------------------------------------------------------
+# Stub kubectl: replays CANI_STUB_OUT on stdout, CANI_STUB_ERR on stderr,
+# and exits CANI_STUB_RC.
+# ---------------------------------------------------------------------------
+# Order matters: kubectl emits warnings during Complete()/Validate() and the
+# verdict later, in RunAccessCheck().  Under the old `2>&1` capture the warning
+# therefore landed IN FRONT of the verdict, which is precisely why `no*` stopped
+# matching.  A stub that printed stdout first would hide the bug it is here to
+# catch, so stderr is written first.
+cat > "$STUB_DIR/kubectl" <<'STUB'
+#!/usr/bin/env bash
+[ -n "${CANI_STUB_ERR:-}" ] && printf '%s\n' "$CANI_STUB_ERR" >&2
+[ -n "${CANI_STUB_OUT:-}" ] && printf '%s\n' "$CANI_STUB_OUT"
+exit "${CANI_STUB_RC:-0}"
+STUB
+chmod +x "$STUB_DIR/kubectl"
+PATH="$STUB_DIR:$PATH"
+
+# shellcheck source=lib/common.sh
+source "$SCRIPT_DIR/lib/common.sh"
+
+SELFTEST_FAIL=0
+
+# ---------------------------------------------------------------------------
+# expect — run one assertion against stubbed kubectl output and check which
+#          counter it landed in.
+#   Usage: expect <label> <stdout> <stderr> <rc> <assert_fn> <want-counter>
+# ---------------------------------------------------------------------------
+expect() {
+  local label="$1" out="$2" err="$3" rc="$4" fn="$5" want="$6"
+
+  GUARD_PASS=0; GUARD_FAIL=0; GUARD_SKIP=0; GUARD_ERROR=0
+  CANI_STUB_OUT="$out" CANI_STUB_ERR="$err" CANI_STUB_RC="$rc" \
+    "$fn" get widgets --as=probe >/dev/null 2>&1
+
+  local got=""
+  [ "$GUARD_PASS"  -eq 1 ] && got="PASS"
+  [ "$GUARD_FAIL"  -eq 1 ] && got="FAIL"
+  [ "$GUARD_SKIP"  -eq 1 ] && got="SKIP"
+  [ "$GUARD_ERROR" -eq 1 ] && got="ERROR"
+
+  if [ "$got" = "$want" ]; then
+    printf '  ok   %-58s -> %s\n' "$label" "$got"
+  else
+    printf '  FAIL %-58s -> %s (wanted %s)\n' "$label" "${got:-none}" "$want"
+    SELFTEST_FAIL=$((SELFTEST_FAIL + 1))
+  fi
+}
+
+# These are kubectl's real strings, not paraphrases.
+WARN_SCOPE="Warning: resource 'clusterrolebindings' is not namespace scoped in group 'rbac.authorization.k8s.io'"
+WARN_UNRESOLVED="Warning: the server doesn't have a resource type 'schedules' in group 'velero.io'"
+ERR_REFUSED="The connection to the server localhost:8080 was refused - did you specify the right host or port?"
+
+printf '=== verdicts on a clean stderr ===\n'
+expect 'cannot / no'                    'no'  ''  1 assert_cannot         PASS
+expect 'cannot / yes (over-scoped)'     'yes' ''  0 assert_cannot         FAIL
+expect 'can / yes'                      'yes' ''  0 assert_can            PASS
+expect 'can / no (under-scoped)'        'no'  ''  1 assert_can            FAIL
+expect 'cannot / no with RBAC reason'   'no - RBAC: denied' '' 1 assert_cannot PASS
+
+printf '\n=== the regression: a benign warning must not eat the verdict ===\n'
+expect 'cannot / no + not-namespace-scoped'  'no'  "$WARN_SCOPE" 1 assert_cannot PASS
+expect 'cannot / yes + not-namespace-scoped' 'yes' "$WARN_SCOPE" 0 assert_cannot FAIL
+expect 'can / yes + not-namespace-scoped'    'yes' "$WARN_SCOPE" 0 assert_can    PASS
+
+printf '\n=== an unresolved resource: the verdict is worthless, not a result ===\n'
+# kubectl answers "no" here, against {group:"", resource:"schedules.velero.io"}.
+# Accepting it would pass every cannot-check vacuously and fail every can-check.
+expect 'cannot_or_skip / unresolved'    'no'  "$WARN_UNRESOLVED" 1 assert_cannot_or_skip SKIP
+expect 'can_or_skip / unresolved'       'no'  "$WARN_UNRESOLVED" 1 assert_can_or_skip    SKIP
+# A core resource missing from discovery is a broken cluster, not a limitation.
+expect 'cannot / unresolved is an error' 'no' "$WARN_UNRESOLVED" 1 assert_cannot         ERROR
+expect 'can / unresolved is an error'    'no' "$WARN_UNRESOLVED" 1 assert_can            ERROR
+
+printf '\n=== infrastructure faults are never filed as a missing CRD ===\n'
+expect 'cannot / connection refused'       '' "$ERR_REFUSED" 1 assert_cannot         ERROR
+expect 'can_or_skip / connection refused'  '' "$ERR_REFUSED" 1 assert_can_or_skip    ERROR
+expect 'cannot_or_skip / connection refused' '' "$ERR_REFUSED" 1 assert_cannot_or_skip ERROR
+expect 'can / empty output, empty stderr'  '' ''             1 assert_can            ERROR
+
+# ---------------------------------------------------------------------------
+# report_total exit codes
+# ---------------------------------------------------------------------------
+printf '\n=== report_total exit codes ===\n'
+check_total() {
+  local label="$1" want="$2"
+  local rc=0
+  report_total >/dev/null 2>&1 || rc=$?
+  if [ "$rc" -eq "$want" ]; then
+    printf '  ok   %-58s -> %d\n' "$label" "$rc"
+  else
+    printf '  FAIL %-58s -> %d (wanted %d)\n' "$label" "$rc" "$want"
+    SELFTEST_FAIL=$((SELFTEST_FAIL + 1))
+  fi
+}
+
+TOTAL_PASS=10 TOTAL_FAIL=0 TOTAL_SKIP=0 TOTAL_ERROR=0; check_total 'all ran and passed'       0
+TOTAL_PASS=10 TOTAL_FAIL=1 TOTAL_SKIP=0 TOTAL_ERROR=0; check_total 'a real failure'           1
+TOTAL_PASS=10 TOTAL_FAIL=0 TOTAL_SKIP=0 TOTAL_ERROR=1; check_total 'a harness error'          3
+TOTAL_PASS=10 TOTAL_FAIL=0 TOTAL_SKIP=1 TOTAL_ERROR=0; check_total 'a skip'                   2
+TOTAL_PASS=0  TOTAL_FAIL=0 TOTAL_SKIP=0 TOTAL_ERROR=0; check_total 'zero checks is not a pass' 2
+TOTAL_PASS=10 TOTAL_FAIL=1 TOTAL_SKIP=1 TOTAL_ERROR=1; check_total 'failure outranks the rest' 1
+
+# ---------------------------------------------------------------------------
+# Subresource syntax lint
+#
+# `kubectl auth can-i <verb> pods/exec` does NOT ask about the exec
+# subresource.  It parses `pods/exec` as resource=pods, name=exec, and the
+# SelfSubjectAccessReview goes out as
+#   {"verb":"create","resource":"pods","name":"exec"}
+# -- verified by inspecting the request body against a live apiserver.
+#
+# So every must-not check written that way was vacuous: measured on
+# kube-apiserver v1.30.6, a service account explicitly granted `pods/exec`
+# create still answered "no" to `can-i create pods/exec`, and "yes" only to
+# `can-i create pods --subresource=exec`.  Nine assertions across the guards
+# were in that state, silently unable to detect the escalation they named.
+#
+# `resource/name` is a legitimate and intentional form here -- it is how the
+# resourceName tests pin a named object (configmaps/coredns,
+# namespaces/kube-system).  Only the names that are real subresources are
+# flagged.
+# ---------------------------------------------------------------------------
+printf '\n=== subresource syntax lint ===\n'
+
+# Real subresources, taken from the discovery document of a live apiserver
+# rather than from memory.
+SUBRESOURCES='proxy|status|exec|log|attach|portforward|binding|eviction|ephemeralcontainers|token|finalize|scale|approval|resize'
+
+LINT_HITS=$(grep -rnE "assert_(can|cannot)(_or_skip)? +[a-z*]+ +[a-z]+/($SUBRESOURCES)\b" \
+  "$SCRIPT_DIR/guards" "$SCRIPT_DIR/lib" 2>/dev/null || true)
+
+if [ -n "$LINT_HITS" ]; then
+  printf '  FAIL resource/subresource written as a name; use --subresource=\n'
+  printf '%s\n' "$LINT_HITS" | sed 's/^/       /'
+  SELFTEST_FAIL=$((SELFTEST_FAIL + 1))
+else
+  printf '  ok   no assertion writes a subresource as a resource name\n'
+fi
+
+printf '\n'
+if [ "$SELFTEST_FAIL" -gt 0 ]; then
+  printf 'HARNESS SELF-TEST FAILED (%d case(s))\n' "$SELFTEST_FAIL" >&2
+  exit 1
+fi
+printf 'harness self-test: all cases behaved correctly\n'

package/tests/validate-agent-frontmatter-schema.py

@@ -13,6 +13,13 @@ Required fields (empirical from current 141-file corpus):
 Optional but typed when present: name, description, model, allowed-tools,
 tools, color. additionalProperties is permitted so harness-specific fields
 do not break validation.
+
+Also checks the markdown harness exports (harnesses/<harness>.agent.md). AGENT.md
+carries only `metadata`, so these exports are where an agent gets its identity,
+and every one must declare a non-empty `name` and `description`. Without a name,
+Claude Code names a plugin agent after its file, and every export here is called
+claude-code.agent.md, so nameless agents collapse into one and the rest are
+dropped as duplicates. For the same reason claude-code names must be unique.
 """
 
 from __future__ import annotations
@@ -211,6 +218,64 @@ def validate_agent(agent_md: Path, schema: dict) -> list[str]:
     return errors
 
 
+# The markdown exports each harness loads as an agent definition. codex.toml and
+# kiro-cli.agent.json carry identity in their own formats and are not covered.
+HARNESS_MARKDOWN = ("claude-code", "cursor", "copilot", "gemini", "kiro-ide")
+
+# Gemini CLI validates local agent frontmatter with a strict schema
+# (localAgentSchema in packages/core/src/agents/agentLoader.ts): `name` must
+# match /^[a-z0-9-_]+$/, unknown keys are rejected, and a file that fails is
+# skipped rather than loaded. The agent id satisfies the slug rule and is unique,
+# so it is what every Gemini export uses; the readable name is `display_name`.
+GEMINI_ALLOWED_KEYS = {
+    "kind", "name", "description", "display_name", "tools", "mcp_servers",
+    "model", "temperature", "max_turns", "timeout_mins",
+}
+
+
+def validate_harness_identity() -> tuple[list[str], int]:
+    """Every markdown harness export names and describes its agent, and no two
+    claude-code exports share a name. Returns (errors, files checked)."""
+    errors: list[str] = []
+    checked = 0
+    claude_names: dict[str, Path] = {}
+    for harness in HARNESS_MARKDOWN:
+        for path in sorted(AGENTS_DIR.glob(f"*/*/harnesses/{harness}.agent.md")):
+            checked += 1
+            rel = path.relative_to(ROOT)
+            fm = parse_frontmatter_raw(path.read_text(encoding="utf-8"))
+            missing = [
+                key for key in ("name", "description")
+                if not (isinstance(fm, dict) and isinstance(fm.get(key), str) and fm[key].strip())
+            ]
+            if missing:
+                errors.append(f"{rel}: missing {' and '.join(missing)} in frontmatter")
+                continue
+            if harness == "gemini":
+                agent_id = path.parent.parent.name
+                if fm["name"] != agent_id:
+                    errors.append(
+                        f"{rel}: name {fm['name']!r} must be the agent id {agent_id!r}; "
+                        "Gemini CLI skips agents whose name is not a slug"
+                    )
+                unknown = sorted(set(fm) - GEMINI_ALLOWED_KEYS)
+                if unknown:
+                    errors.append(
+                        f"{rel}: frontmatter keys {unknown} are not in Gemini CLI's "
+                        "local agent schema, which rejects unknown keys"
+                    )
+                if fm.get("kind", "local") != "local":
+                    errors.append(f"{rel}: kind {fm['kind']!r} is not 'local'")
+            if harness == "claude-code":
+                prior = claude_names.setdefault(fm["name"], path)
+                if prior != path:
+                    errors.append(
+                        f"{rel}: name {fm['name']!r} is already used by "
+                        f"{prior.relative_to(ROOT)}; Claude Code would drop one"
+                    )
+    return errors, checked
+
+
 # ──────────────────────────────────────────────────────────────────────────────
 # Entry point
 # ──────────────────────────────────────────────────────────────────────────────
@@ -234,6 +299,8 @@ def main() -> int:
         if errors:
             failed.append((agent_md, errors))
 
+    harness_errors, harness_checked = validate_harness_identity()
+
     if failed:
         print(
             f"FAIL: {len(failed)} agent(s) failed AGENT.md frontmatter schema validation "
@@ -244,11 +311,24 @@ def main() -> int:
             print(f"\n  {path}", file=sys.stderr)
             for e in errs:
                 print(f"    {e}", file=sys.stderr)
+    if harness_errors:
+        print(
+            f"FAIL: {len(harness_errors)} harness export(s) lack a usable agent "
+            f"identity (out of {harness_checked} checked):",
+            file=sys.stderr,
+        )
+        for e in harness_errors:
+            print(f"  {e}", file=sys.stderr)
+    if failed or harness_errors:
         return 1
 
     print(
         f"OK: all {len(agent_files)} agents passed AGENT.md frontmatter schema validation"
     )
+    print(
+        f"OK: {harness_checked} markdown harness exports declare name and description "
+        f"(claude-code names unique; Gemini names are agent ids)"
+    )
     return 0
 
 

package/tests/validate-mcp-trust-matrix.py

@@ -1,5 +1,5 @@
 #!/usr/bin/env python3
-"""Validate the trust_matrix block on every MCP reference metadata file.
+"""Validate the trust_matrix block on every MCP reference, source and catalog.
 
 The trust_matrix field is optional in the schema today (graceful rollout),
 but this validator enforces it on every file currently committed under
@@ -10,6 +10,14 @@ posture explicitly.
 Promotes to a required field in the schema once the corpus and
 contribution guide have been updated. Until then, this validator is the
 de-facto contract.
+
+It also enforces that catalog/mcp-references.json carries the SAME
+trust_matrix as the mcp/ source metadata. That index is hand-maintained and
+had lost the field entirely: the Rust console deserialises it with
+`Option<TrustMatrix>` and treats `None` as "no boundary to enforce", so a
+missing trust_matrix in the catalog silently made every MCP pass every trust
+check no matter what its source metadata declared. Source and index must
+agree, or the enforcement path is decorative.
 """
 
 from __future__ import annotations
@@ -40,6 +48,18 @@ def metadata_files() -> list[Path]:
     return sorted(set(paths))
 
 
+CATALOG_INDEX = ROOT / "catalog" / "mcp-references.json"
+
+
+def catalog_entries() -> dict[str, dict]:
+    """Map id -> entry from the hand-maintained catalog index."""
+    if not CATALOG_INDEX.exists():
+        return {}
+    data = json.loads(CATALOG_INDEX.read_text(encoding="utf-8"))
+    items = data if isinstance(data, list) else data.get("mcp_references", [])
+    return {e["id"]: e for e in items if isinstance(e, dict) and e.get("id")}
+
+
 def main() -> int:
     files = metadata_files()
     if not files:
@@ -77,13 +97,53 @@ def main() -> int:
                 f"{rel}: trust_matrix.pin_strategy must be one of {sorted(ALLOWED_PIN)}"
             )
 
+    # Source <-> catalog parity. The console reads only the catalog index, so a
+    # trust_matrix that exists in mcp/ but not here is not enforced anywhere.
+    catalog = catalog_entries()
+    checked_parity = 0
+    for path in files:
+        try:
+            data = json.loads(path.read_text(encoding="utf-8"))
+        except Exception:  # noqa: BLE001
+            continue  # already reported above
+        mcp_id = data.get("id")
+        source_tm = data.get("trust_matrix")
+        if not mcp_id or not isinstance(source_tm, dict):
+            continue
+        entry = catalog.get(mcp_id)
+        if entry is None:
+            errors.append(
+                f"catalog/mcp-references.json: no entry for MCP id {mcp_id!r} "
+                f"declared in {path.relative_to(ROOT)}"
+            )
+            continue
+        catalog_tm = entry.get("trust_matrix")
+        if not isinstance(catalog_tm, dict):
+            errors.append(
+                f"catalog/mcp-references.json: entry {mcp_id!r} is missing trust_matrix. "
+                f"The console treats a missing matrix as 'nothing to enforce', so this "
+                f"MCP would pass every trust-boundary check. Copy it from "
+                f"{path.relative_to(ROOT)}."
+            )
+            continue
+        if catalog_tm != source_tm:
+            errors.append(
+                f"catalog/mcp-references.json: trust_matrix for {mcp_id!r} does not match "
+                f"{path.relative_to(ROOT)} (source={source_tm}, catalog={catalog_tm})"
+            )
+            continue
+        checked_parity += 1
+
     if errors:
         print("ERROR: MCP trust_matrix validation failed", file=sys.stderr)
         for err in errors:
             print(f"  - {err}", file=sys.stderr)
         return 1
 
-    print(f"OK: validated trust_matrix on {len(files)} MCP reference files")
+    print(
+        f"OK: validated trust_matrix on {len(files)} MCP reference files "
+        f"({checked_parity} in source/catalog parity)"
+    )
     return 0
 
 

package/tests/validate-release-notes.mjs

@@ -1,17 +1,94 @@
+import fs from "node:fs";
+import path from "node:path";
 import { createRequire } from "node:module";
-import { generateNotes } from "@semantic-release/release-notes-generator";
+import { pathToFileURL } from "node:url";
 
 const require = createRequire(import.meta.url);
 const config = require("../.releaserc.js");
-const pluginConfig = config.plugins.find(
-  ([name]) => name === "@semantic-release/release-notes-generator",
-)?.[1];
 
-if (!pluginConfig) {
-  throw new Error("release-notes-generator configuration is missing");
+// Resolve each plugin exactly as semantic-release does: from its own
+// lib/plugins directory first, falling back to the project
+// (semantic-release/lib/plugins/utils.js, loadPlugin). Importing the bare
+// specifier from here would resolve from the project root instead, so if npm
+// ever nested a different plugin copy under semantic-release, this gate would
+// pass against a module the release never runs.
+const semanticReleaseDir = path.dirname(require.resolve("semantic-release"));
+const loadPluginRequire = createRequire(
+  path.join(semanticReleaseDir, "lib", "plugins", "utils.js"),
+);
+
+function resolvePlugin(name) {
+  try {
+    return loadPluginRequire.resolve(name);
+  } catch {
+    return require.resolve(name);
+  }
+}
+
+// Which copy of `name` Node would load for code in `fromFile`: the first
+// node_modules/<name> walking up from its directory.
+function installedVersion(fromFile, name) {
+  for (let dir = path.dirname(fromFile); ; dir = path.dirname(dir)) {
+    const manifest = path.join(dir, "node_modules", name, "package.json");
+    if (fs.existsSync(manifest)) {
+      return JSON.parse(fs.readFileSync(manifest, "utf8")).version;
+    }
+    if (path.dirname(dir) === dir) {
+      throw new Error(`${name} is not resolvable from ${fromFile}`);
+    }
+  }
+}
+
+function pluginConfigFor(name) {
+  const entry = config.plugins.find(([pluginName]) => pluginName === name)?.[1];
+  if (!entry) {
+    throw new Error(`${name} configuration is missing`);
+  }
+  return entry;
+}
+
+const failures = [];
+const quietLogger = { log() {}, warn() {}, error() {}, success() {} };
+
+// conventional-changelog-conventionalcommits@10 renders only through
+// conventional-changelog-writer@9 and refuses older writers outright. The
+// stable plugins declare writer@8 and parser@6; package.json `overrides`
+// raise them to the set their own next majors publish. Assert the copies the
+// release will actually load, so a lockfile or override regression names
+// itself instead of surfacing as a template error at release time.
+const presetMajor = Number(
+  installedVersion(require.resolve("../package.json"), "conventional-changelog-conventionalcommits").split(".")[0],
+);
+const REQUIRED_MAJORS = { "conventional-changelog-writer": 9, "conventional-commits-parser": 7 };
+const pluginFiles = {};
+for (const name of ["@semantic-release/commit-analyzer", "@semantic-release/release-notes-generator"]) {
+  pluginFiles[name] = resolvePlugin(name);
+  if (presetMajor < 10) continue;
+  for (const [dep, major] of Object.entries(REQUIRED_MAJORS)) {
+    const version = installedVersion(pluginFiles[name], dep);
+    if (Number(version.split(".")[0]) < major) {
+      failures.push(
+        `${name} would load ${dep}@${version}; conventionalcommits@${presetMajor} needs ${major}+`,
+      );
+    }
+  }
 }
 
-const notes = await generateNotes(pluginConfig, {
+// Report a wrong writer or parser before rendering: the preset's own guard
+// would otherwise throw a template error that hides which copy was loaded.
+if (failures.length) {
+  for (const failure of failures) console.error(`FAIL ${failure}`);
+  process.exit(1);
+}
+
+const { generateNotes } = await import(
+  pathToFileURL(pluginFiles["@semantic-release/release-notes-generator"]).href
+);
+const { analyzeCommits } = await import(
+  pathToFileURL(pluginFiles["@semantic-release/commit-analyzer"]).href
+);
+
+const notes = await generateNotes(pluginConfigFor("@semantic-release/release-notes-generator"), {
   commits: [
     {
       hash: "1234567890abcdef",
@@ -29,7 +106,45 @@ const notes = await generateNotes(pluginConfig, {
 });
 
 if (!notes.includes("v3.12.0") || !notes.includes("evidence-bounded review suite")) {
-  throw new Error("release notes are missing the expected version or feature");
+  failures.push("release notes are missing the expected version or feature");
+}
+
+// The analyzer decides the version bump, so a preset or parser change that
+// misreads a commit ships the wrong version with no error. One commit per
+// case, against the configured releaseRules.
+const ANALYZER_CASES = [
+  ["feat(api): add endpoint", "minor"],
+  ["fix(api): handle empty body", "patch"],
+  ["feat(api)!: drop the v1 routes", "major"],
+  ["fix(api): tighten parsing\n\nBREAKING CHANGE: the v1 payload is rejected", "major"],
+  ["security(auth): rotate the signing key", "patch"],
+  ["perf(export): stream large catalogs", "patch"],
+  ["refactor(exporter): split the writer", "patch"],
+  ["build(deps): bump fast-check", "patch"],
+  ["revert: feat(api): add endpoint", "patch"],
+  ["docs(readme): fix a link", null],
+  ["chore(deps): bump the lockfile", null],
+  ["feat(no-release): internal tooling only", null],
+];
+const analyzerConfig = pluginConfigFor("@semantic-release/commit-analyzer");
+for (const [message, expected] of ANALYZER_CASES) {
+  const got = await analyzeCommits(analyzerConfig, {
+    commits: [{ hash: "1234567890abcdef", message }],
+    cwd: process.cwd(),
+    logger: quietLogger,
+  });
+  if ((got ?? null) !== expected) {
+    failures.push(
+      `commit-analyzer: ${JSON.stringify(message.split("\n")[0])} -> ${got ?? null}, expected ${expected}`,
+    );
+  }
+}
+
+if (failures.length) {
+  for (const failure of failures) console.error(`FAIL ${failure}`);
+  process.exit(1);
 }
 
-console.log("OK: semantic-release notes generated with the configured preset");
+console.log(
+  `OK: semantic-release notes generated and ${ANALYZER_CASES.length} commit-analysis cases hold with conventionalcommits@${presetMajor}`,
+);