aero-agent-roles 1.2.2 → 1.2.3

package/roles/software-product-assurance-engineer/tests/test_role_software_product_assurance_engineer.py

@@ -0,0 +1,202 @@
+#!/usr/bin/env python3
+"""Role test: Software Product Assurance Engineer (ECSS-Q-ST-80C).
+
+Verifies (offline, no network):
+1. Every skill bound in ROLE.md frontmatter RESOLVES to a real leaf in the
+   aero-agent-skills repo, and every bound leaf ships a logic file the
+   role can dispatch.
+2. The workflow stage order is deterministic and complete.
+3. The filled deliverable template carries no blanks, the draft and
+   no-approval markers, ends at the stop line, and is byte-identical to a
+   fresh build of the bundled worked example (no stale template).
+4. The role ships the 100% anatomy (core engine, cli, SOURCES, example),
+   spells out its abbreviations, carries no machine paths, and the author
+   is ashfordeOU.
+"""
+import os
+import re
+import sys
+import unittest
+
+ROLES_REPO = os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(
+    os.path.abspath(__file__)))))          # repo root (4 up from tests/)
+# Bound-skill resolution is a cross-repo dev check: it runs when
+# AEROSKILLS_DEV points at an aero-agent-skills checkout, and skips
+# otherwise (role-lint resolves the same list against the pinned ledger).
+AEROSKILLS = os.environ.get("AEROSKILLS_DEV", "")
+HAS_SKILLS = bool(AEROSKILLS) and os.path.isdir(os.path.join(AEROSKILLS,
+                                                             "skills"))
+ROLE_DIR = os.path.join(ROLES_REPO, "roles",
+                        "software-product-assurance-engineer")
+TEMPLATE = os.path.join(ROLE_DIR, "templates",
+                        "q80-compliance-matrix-template.md")
+sys.path.insert(0, os.path.join(ROLE_DIR, "core"))
+
+EXPECTED_BOUND = [
+    "space-systems/ecss/q80-software-criticality-tailoring",
+    "space-systems/ecss/q80-compliance-matrix",
+    "space-systems/ecss/q80-milestone-assurance-evidence",
+    "space-systems/ecss/q80-software-product-assurance-plan",
+    "space-systems/ecss/q80-software-process-assurance",
+    "space-systems/ecss/q80-software-product-quality-metrics",
+    "space-systems/ecss/software-engineering",
+    "space-systems/ecss/software-verification",
+]
+
+EXPECTED_STAGES = [
+    "1. Category", "2. Tailoring", "3. Evidence index",
+    "4. Compliance matrix", "5. Document trace", "6. Gap routing",
+    "7. Milestone evidence", "8. Milestone report skeleton",
+    "9. Product metrics", "10. Engineering context", "11. Stop gate",
+]
+
+ABBREVIATIONS = {
+    "PA": "product assurance",
+    "ECSS": "European Cooperation for Space Standardization",
+    "SRR": "system requirements review",
+    "PDR": "preliminary design review",
+    "CDR": "critical design review",
+    "QR": "qualification review",
+    "AR": "acceptance review",
+    "TRR": "test readiness review",
+    "ORR": "operational readiness review",
+    "SPAP": "software product assurance plan",
+    "SPAMR": "software product assurance milestone report",
+    "CSV": "comma-separated values",
+}
+
+
+def read_role_md():
+    with open(os.path.join(ROLE_DIR, "ROLE.md")) as f:
+        return f.read()
+
+
+def body_of(text):
+    """The ROLE.md body with line wraps folded to single spaces."""
+    return re.sub(r"\s+", " ", text.split("\n---\n", 1)[1])
+
+
+class TestSoftwareProductAssuranceRole(unittest.TestCase):
+
+    def test_bound_skills_resolve_in_aeroskills(self):
+        if not HAS_SKILLS:
+            self.skipTest("Aero Agent Skills checkout not present "
+                          "(cross-repo dev check)")
+        for leaf in EXPECTED_BOUND:
+            sk = os.path.join(AEROSKILLS, "skills", leaf, "SKILL.md")
+            self.assertTrue(os.path.exists(sk),
+                            "bound skill not found: %s" % leaf)
+            scripts = os.path.join(AEROSKILLS, "skills", leaf, "scripts")
+            logic = [f for f in os.listdir(scripts)
+                     if f.endswith("_logic.py") and not f.startswith("test_")]
+            self.assertTrue(logic, "bound leaf has no logic: %s" % leaf)
+
+    def test_bound_skills_are_exactly_the_frontmatter_list(self):
+        fm = read_role_md().split("\n---\n", 1)[0]
+        block = fm.split("skills_bound:", 1)[1].split("\ntools_allowed:")[0]
+        listed = re.findall(r"^\s+-\s+(\S+)\s*$", block, re.M)
+        self.assertEqual(listed, EXPECTED_BOUND)
+
+    def test_workflow_stages_deterministic(self):
+        role = read_role_md()
+        wf = role.split("## Workflow", 1)[1].split("\n## ", 1)[0]
+        idx = [wf.find("| " + stage + " |") for stage in EXPECTED_STAGES]
+        self.assertTrue(all(i >= 0 for i in idx),
+                        "a stage is missing: %s" % idx)
+        self.assertEqual(idx, sorted(idx), "stage order not deterministic")
+
+    def test_every_bound_leaf_has_a_stage(self):
+        wf = read_role_md().split("## Workflow", 1)[1].split("\n## ", 1)[0]
+        for leaf in EXPECTED_BOUND:
+            self.assertIn(leaf.rsplit("/", 1)[-1], wf,
+                          "no workflow stage loads %s" % leaf)
+
+    def test_deliverable_template_complete(self):
+        with open(TEMPLATE) as f:
+            text = f.read()
+        for n in range(1, 12):
+            self.assertTrue(re.search(r"^## %d\. " % n, text, re.M),
+                            "section %d missing" % n)
+        self.assertEqual(len(re.findall(r"___|TODO|TBD", text)), 0,
+                         "template must be a FILLED deliverable (0 blanks)")
+        low = text.lower()
+        for marker in ("draft", "not an approval", "not a certification",
+                       "not a statement of compliance", "ecss-q-st-80c rev.2"):
+            self.assertIn(marker, low)
+        self.assertEqual(text.rstrip().splitlines()[-1],
+                         "STOP: human sign-off required before submission.")
+
+    def test_template_is_a_fresh_build_of_the_example(self):
+        import software_product_assurance_core as core
+        with open(TEMPLATE) as f:
+            self.assertEqual(f.read(), core.example_report_markdown(),
+                             "template is stale: regenerate it with "
+                             "python3 cli.py build --no-dispatch > "
+                             "templates/q80-compliance-matrix-template.md")
+
+    def test_forbidden_lines_present(self):
+        role = read_role_md()
+        for phrase in ["mark a compliance matrix approved or signed",
+                       "issue a statement of compliance",
+                       "sign_off_required: true",
+                       "reproduce proprietary standard text"]:
+            self.assertIn(phrase, role, "missing boundary: %s" % phrase)
+
+    def test_abbreviations_spelled_out_on_first_use(self):
+        body = body_of(read_role_md())
+        for abbr, words in ABBREVIATIONS.items():
+            # a standard's designation (ECSS-Q-ST-80C) is a name, not a use
+            first = re.search(r"\b%s\b(?!-)" % abbr, body)
+            self.assertIsNotNone(first, "%s not used" % abbr)
+            spelled = body.lower().find(words.lower())
+            self.assertGreaterEqual(spelled, 0, "%s never spelled out" % abbr)
+            self.assertLess(spelled, first.start(),
+                            "%s used before it is spelled out" % abbr)
+
+    def test_sources_register_exists(self):
+        with open(os.path.join(ROLE_DIR, "SOURCES.md")) as f:
+            text = f.read()
+        self.assertIn("ECSS-Q-ST-80C Rev.2", text)
+        self.assertIn("paraphrased", text)
+
+    def test_core_engine_and_example_exist(self):
+        for f in ["core/software_product_assurance_core.py", "cli.py",
+                  "templates", "example/evidence.csv", "example/docs",
+                  "example/pdr-pack.csv", "example/metrics.csv"]:
+            self.assertTrue(os.path.exists(os.path.join(ROLE_DIR, f)),
+                            "%s missing" % f)
+
+    def test_example_files_are_marked(self):
+        docs = os.path.join(ROLE_DIR, "example", "docs")
+        for name in os.listdir(docs):
+            self.assertTrue(name.startswith("EX-"), name)
+            with open(os.path.join(docs, name)) as f:
+                self.assertIn("EXAMPLE STUB", f.read())
+
+    def test_no_machine_paths_shipped(self):
+        # built from pieces so this file does not match its own pattern
+        pat = re.compile("|".join(["/" + "Users/", "/" + "home/[a-z]",
+                                   "/" + "Volumes/"]))
+        for root, _dirs, files in os.walk(ROLE_DIR):
+            if "__pycache__" in root:
+                continue
+            for name in files:
+                if name.endswith((".md", ".csv", ".py")):
+                    with open(os.path.join(root, name)) as f:
+                        self.assertIsNone(pat.search(f.read()),
+                                          "machine path in %s" % name)
+
+    def test_author_ashfordeou(self):
+        self.assertIn("author: ashfordeOU", read_role_md())
+
+    def test_no_approval_claims_in_template(self):
+        with open(TEMPLATE) as f:
+            low = f.read().lower()
+        for phrase in ["signed-off-by", "status: approved", "status: signed",
+                       "is compliant with ecss"]:
+            self.assertNotIn(phrase, low,
+                             "template must not claim approval: %s" % phrase)
+
+
+if __name__ == "__main__":
+    unittest.main()

package/roles/software-product-assurance-engineer/tests/test_software_product_assurance_cli.py

@@ -0,0 +1,271 @@
+#!/usr/bin/env python3
+"""CLI test: software-product-assurance-engineer build / check end to end.
+
+Runs cli.py as a user would (offline, no network): the bundled example,
+a real-project run from a temporary evidence file and document folder,
+the input errors, the evidence bundle and profile header, the check of a
+draft and of a human-signed matrix, and the cross-check against the bound
+q80 leaves (dispatched when AEROSKILLS_DEV points at the skills, and shown
+to catch a disagreement; otherwise the not-dispatched message must say
+how to point AEROSKILLS_DEV at an installed aero-agent-skills package).
+"""
+import json
+import os
+import shutil
+import subprocess
+import sys
+import tempfile
+import unittest
+
+ROLE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+CLI = os.path.join(ROLE_DIR, "cli.py")
+REPO_ROOT = os.path.dirname(os.path.dirname(ROLE_DIR))
+PROFILE = os.path.join(REPO_ROOT, "profiles", "example-airframer.json")
+SKILLS = os.environ.get("AEROSKILLS_DEV", "")
+HAS_Q80 = bool(SKILLS) and os.path.isdir(os.path.join(
+    SKILLS, "skills", "space-systems", "ecss", "q80-compliance-matrix"))
+STOP = "STOP: human sign-off required before submission."
+
+EVIDENCE = (
+    "clause,document,section,status,justification\n"
+    "5.3.1,PLAN-001,section 6,compliant,\n"
+    "5.2.3,PLAN-001,section 4,partially-compliant,audits start at CDR\n"
+    "6.2.6.5,REPORT-009,section 3,compliant,\n"
+    "7.1.1,,,compliant,in the specification\n")
+
+
+def run(args, env_skills=None, cwd=None):
+    env = dict(os.environ)
+    if env_skills is not None:
+        env["AEROSKILLS_DEV"] = env_skills
+    return subprocess.run([sys.executable, CLI] + args, capture_output=True,
+                          text=True, env=env, cwd=cwd)
+
+
+class CliCase(unittest.TestCase):
+
+    def setUp(self):
+        self.tmp = tempfile.mkdtemp()
+
+    def tearDown(self):
+        shutil.rmtree(self.tmp, ignore_errors=True)
+
+    def path(self, *p):
+        return os.path.join(self.tmp, *p)
+
+    def write(self, name, text):
+        with open(self.path(name), "w") as f:
+            f.write(text)
+        return self.path(name)
+
+
+class TestBuildExample(CliCase):
+
+    def test_example_builds_markdown_and_csv(self):
+        r = run(["build", "--out", self.path("out", "matrix.md")],
+                env_skills="/nonexistent")
+        self.assertEqual(r.returncode, 0, r.stdout + r.stderr)
+        self.assertTrue(os.path.exists(self.path("out", "matrix.csv")))
+        self.assertIn("worked example", r.stdout)
+        self.assertEqual(r.stdout.rstrip().splitlines()[-1], STOP)
+        with open(self.path("out", "matrix.md")) as f:
+            self.assertEqual(f.read().rstrip().splitlines()[-1], STOP)
+
+    def test_check_passes_on_a_fresh_build(self):
+        run(["build", "--out", self.path("m.md")], env_skills="/nonexistent")
+        r = run(["check", "--file", self.path("m.md")])
+        self.assertEqual(r.returncode, 0, r.stdout)
+        self.assertIn("SIGN-OFF: none recorded", r.stdout)
+        self.assertIn("RESULT: PASS", r.stdout)
+
+    def test_stdout_mode_prints_the_matrix(self):
+        r = run(["build", "--no-dispatch"])
+        self.assertEqual(r.returncode, 0, r.stderr)
+        self.assertTrue(r.stdout.startswith("# Software Product Assurance "
+                                            "Compliance Matrix"))
+
+    def test_bundle_records_honest_provenance(self):
+        r = run(["build", "--out", self.path("m.md"), "--bundle"],
+                env_skills="/nonexistent")
+        self.assertEqual(r.returncode, 0, r.stdout + r.stderr)
+        with open(self.path("evidence", "provenance.json")) as f:
+            prov = json.load(f)
+        self.assertFalse(prov["cross_checked"])
+        self.assertEqual(prov["skills"], [])
+        self.assertIn("AEROSKILLS_DEV", prov["skills_not_dispatched_reason"])
+        with open(self.path("evidence", "model.json")) as f:
+            model = json.load(f)
+        self.assertEqual(model["status"], "DRAFT")
+        self.assertTrue(model["requires_human_sign_off"])
+
+    def test_profile_header(self):
+        if not os.path.exists(PROFILE):
+            self.skipTest("profiles/example-airframer.json not present")
+        r = run(["build", "--out", self.path("m.md"), "--profile", PROFILE],
+                env_skills="/nonexistent")
+        self.assertEqual(r.returncode, 0, r.stdout + r.stderr)
+        with open(self.path("m.md")) as f:
+            self.assertTrue(f.read().startswith("## Program context"))
+
+
+class TestBuildProject(CliCase):
+
+    def test_real_project_with_trace(self):
+        ev = self.write("ev.csv", EVIDENCE)
+        os.makedirs(self.path("docs", "plans"))
+        self.write(os.path.join("docs", "plans", "PLAN-001.docx"), "x")
+        r = run(["build", "--out", self.path("m.md"), "--evidence", ev,
+                 "--category", "C", "--docs", self.path("docs"),
+                 "--review", "CDR", "--project", "Test craft"],
+                env_skills="/nonexistent")
+        self.assertEqual(r.returncode, 0, r.stdout + r.stderr)
+        self.assertIn("1 missing (REPORT-009)", r.stdout)
+        with open(self.path("m.md")) as f:
+            md = f.read()
+        self.assertIn("| REPORT-009 | 6.2.6.5 |", md)
+        self.assertIn("Project: Test craft", md)
+        self.assertIn("Evidence index: ev.csv (4 rows)", md)
+        self.assertIn("Document folder: docs/ (1 files)", md)
+        self.assertNotIn(self.tmp, md, "absolute path leaked into the matrix")
+        self.assertEqual(run(["check", "--file", self.path("m.md")])
+                         .returncode, 0)
+
+    def test_severity_derives_the_category(self):
+        ev = self.write("ev.csv", EVIDENCE)
+        r = run(["build", "--out", self.path("m.md"), "--evidence", ev,
+                 "--severity", "II", "--provision", "operational-procedure"],
+                env_skills="/nonexistent")
+        self.assertEqual(r.returncode, 0, r.stdout + r.stderr)
+        self.assertIn("software category C (derived from severity II)",
+                      r.stdout)
+        self.assertIn("document trace: not run", r.stdout)
+
+    def test_customer_clause_list(self):
+        ev = self.write("ev.csv", EVIDENCE)
+        cl = self.write("list.csv", "clause\n5.3.1\n5.2.3\n")
+        r = run(["build", "--out", self.path("m.md"), "--evidence", ev,
+                 "--category", "B", "--clauses", cl],
+                env_skills="/nonexistent")
+        self.assertEqual(r.returncode, 0, r.stdout + r.stderr)
+        self.assertIn("Coverage: 2 clauses", r.stdout)
+
+    def test_missing_category_is_an_input_error(self):
+        ev = self.write("ev.csv", EVIDENCE)
+        r = run(["build", "--out", self.path("m.md"), "--evidence", ev])
+        self.assertEqual(r.returncode, 2)
+        self.assertIn("--category", r.stderr)
+
+    def test_bad_status_is_an_input_error(self):
+        ev = self.write("ev.csv", "clause,status\n5.3.1,mostly\n")
+        r = run(["build", "--out", self.path("m.md"), "--evidence", ev,
+                 "--category", "B"])
+        self.assertEqual(r.returncode, 2)
+        self.assertIn("unknown compliance status", r.stderr)
+
+    def test_missing_files_are_input_errors(self):
+        r = run(["build", "--evidence", self.path("none.csv"),
+                 "--category", "B"])
+        self.assertEqual(r.returncode, 2)
+        ev = self.write("ev.csv", EVIDENCE)
+        r = run(["build", "--evidence", ev, "--category", "B", "--docs",
+                 self.path("no-docs")])
+        self.assertEqual(r.returncode, 2)
+        r = run(["build", "--evidence", ev, "--category", "B", "--pack", ev])
+        self.assertEqual(r.returncode, 2)
+        self.assertIn("--review", r.stderr)
+
+
+class TestCheck(CliCase):
+
+    def built(self):
+        run(["build", "--out", self.path("m.md")], env_skills="/nonexistent")
+        with open(self.path("m.md")) as f:
+            return f.read()
+
+    def test_human_signed_matrix_passes(self):
+        md = self.built()
+        gaps = [l for l in md.splitlines() if l.startswith("Open gaps: ")][0]
+        n = gaps.split()[2].rstrip(".")
+        self.write("m.md", md + "Signed-off-by: A. Person\n"
+                   "Sign-off-role: software PA manager\n"
+                   "Sign-off-date: 2026-10-01\nSign-off-decision: approved\n"
+                   "Accepted-open-gaps: %s\n" % n)
+        r = run(["check", "--file", self.path("m.md")])
+        self.assertEqual(r.returncode, 0, r.stdout)
+        self.assertIn("SIGN-OFF: approved by A. Person", r.stdout)
+
+    def test_signature_without_gap_acceptance_fails(self):
+        md = self.built()
+        self.write("m.md", md + "Signed-off-by: A. Person\n"
+                   "Sign-off-role: software PA manager\n"
+                   "Sign-off-date: 2026-10-01\nSign-off-decision: approved\n")
+        r = run(["check", "--file", self.path("m.md")])
+        self.assertEqual(r.returncode, 1)
+        self.assertIn("sign-off problem", r.stdout)
+
+    def test_tampered_matrix_fails(self):
+        md = self.built().replace(STOP, "")
+        self.write("m.md", md)
+        r = run(["check", "--file", self.path("m.md")])
+        self.assertEqual(r.returncode, 1)
+        self.assertIn("FAIL  sign_off_honest", r.stdout)
+
+    def test_missing_file(self):
+        self.assertEqual(run(["check", "--file", self.path("x.md")])
+                         .returncode, 1)
+
+
+class TestCrossCheck(CliCase):
+
+    def test_not_dispatched_says_how_to_point_at_the_skills(self):
+        r = run(["build", "--out", self.path("m.md")], env_skills="")
+        self.assertEqual(r.returncode, 0)
+        self.assertIn("not dispatched (AEROSKILLS_DEV is not set)", r.stdout)
+        self.assertIn('export AEROSKILLS_DEV="$(npm root)/aero-agent-skills"',
+                      r.stdout)
+
+    def test_required_cross_check_fails_loud_without_skills(self):
+        r = run(["build", "--out", self.path("m.md"),
+                 "--require-cross-check"], env_skills="/nonexistent")
+        self.assertEqual(r.returncode, 2)
+        self.assertIn("npm install aero-agent-skills", r.stderr)
+        self.assertFalse(os.path.exists(self.path("m.md")))
+
+    def test_dispatch_agrees_with_the_bound_leaves(self):
+        if not HAS_Q80:
+            self.skipTest("AEROSKILLS_DEV with the q80 leaves not present")
+        r = run(["build", "--out", self.path("m.md"), "--bundle",
+                 "--require-cross-check"])
+        self.assertEqual(r.returncode, 0, r.stdout + r.stderr)
+        with open(self.path("evidence", "provenance.json")) as f:
+            prov = json.load(f)
+        self.assertTrue(prov["cross_checked"])
+        leaves = set(row["leaf"].rsplit("/", 1)[-1] for row in prov["skills"])
+        for leaf in ("q80-compliance-matrix",
+                     "q80-software-criticality-tailoring",
+                     "q80-milestone-assurance-evidence",
+                     "q80-software-product-assurance-plan",
+                     "q80-software-product-quality-metrics"):
+            self.assertIn(leaf, leaves)
+
+    def test_dispatch_catches_a_disagreement(self):
+        if not HAS_Q80:
+            self.skipTest("AEROSKILLS_DEV with the q80 leaves not present")
+        sys.path.insert(0, ROLE_DIR)
+        import cli  # noqa: E402
+        core = cli.core
+        item = core.example_item()
+        model = core.build_report(item)
+        _root, mods, _reason = cli.skills_status()
+        clean = cli.dispatch_rows(item, model, mods)
+        self.assertTrue(clean and all(r["agrees"] for r in clean))
+        model["metrics"]["verdict"] = "pass"          # a wrong engine answer
+        model["tailoring"]["totals"]["reduced"] += 1
+        bad = [r for r in cli.dispatch_rows(item, model, mods)
+               if not r["agrees"]]
+        self.assertEqual(sorted(r["function"] for r in bad),
+                         ["evaluate_metrics", "tailoring_matrix"])
+
+
+if __name__ == "__main__":
+    unittest.main()