@christang/keel 5.20.0 → 5.39.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +28 -9
- package/assets/bootstrap/AGENTS.md +1 -1
- package/assets/openspec/schemas/keel-spec-driven/templates/tasks.md +4 -1
- package/bin/keel.js +127 -22
- package/package.json +1 -1
- package/plugins/keel/.claude-plugin/plugin.json +1 -1
- package/plugins/keel/.codex-plugin/plugin.json +1 -1
- package/plugins/keel/skills/keel-align-expectations/SKILL.md +4 -18
- package/plugins/keel/skills/keel-review-checklist/SKILL.md +1 -1
- package/scripts/install_to_repo.py +45 -2
- package/scripts/validate_plugin.py +3120 -204
- package/src/core/config.js +192 -26
- package/src/core/context.js +9 -0
- package/src/core/gates.js +257 -40
- package/src/core/guard.js +77 -16
- package/src/core/task-contract.js +85 -3
|
@@ -37,8 +37,8 @@ REQUIRED_SCRIPTS = [
|
|
|
37
37
|
"scripts/validate_plugin.py",
|
|
38
38
|
]
|
|
39
39
|
|
|
40
|
-
PACKAGE_VERSION = "5.
|
|
41
|
-
PROTOCOL_VERSION = "5.
|
|
40
|
+
PACKAGE_VERSION = "5.39.0"
|
|
41
|
+
PROTOCOL_VERSION = "5.39.0"
|
|
42
42
|
LEGACY_MANAGED_START = "<!-- keel:start version=2.1 -->"
|
|
43
43
|
OPENSPEC_SCHEMA_NAME = "keel-spec-driven"
|
|
44
44
|
# Mirrors KEEL_PACKAGE_NAME in scripts/install_to_repo.py, one of the two
|
|
@@ -1611,6 +1611,118 @@ def validate_expectation_slice_gates_scenario() -> int:
|
|
|
1611
1611
|
return 0
|
|
1612
1612
|
|
|
1613
1613
|
|
|
1614
|
+
def validate_unparsed_covers_critical_statement_scenario() -> int:
|
|
1615
|
+
"""Issue #49 item 1: a critical-statement Covers reference must be told apart
|
|
1616
|
+
as missing (the identifier never appears in design.md) from unparsed (the
|
|
1617
|
+
identifier appears but not in the exact resolvable shape), instead of both
|
|
1618
|
+
collapsing into the same "Missing" message.
|
|
1619
|
+
"""
|
|
1620
|
+
task = task_capsule_compact_fixture().replace(
|
|
1621
|
+
" - E1: Public behavior passes.\n",
|
|
1622
|
+
" - D2\n",
|
|
1623
|
+
)
|
|
1624
|
+
|
|
1625
|
+
def covers_message(repo: Path) -> tuple[int, str]:
|
|
1626
|
+
write_text(repo / "openspec/changes/demo/tasks.md", task)
|
|
1627
|
+
result = run_keel(
|
|
1628
|
+
repo, "gate", "task-start", "--change", "demo", "--task", "1.1", "--json"
|
|
1629
|
+
)
|
|
1630
|
+
problems = json.loads(result.stdout).get("problems", [])
|
|
1631
|
+
message = next(
|
|
1632
|
+
(
|
|
1633
|
+
item.get("message", "")
|
|
1634
|
+
for item in problems
|
|
1635
|
+
if item.get("code") == "unresolved-covers"
|
|
1636
|
+
),
|
|
1637
|
+
"",
|
|
1638
|
+
)
|
|
1639
|
+
return result.returncode, message
|
|
1640
|
+
|
|
1641
|
+
with tempfile.TemporaryDirectory(prefix="keel-unparsed-covers-missing-") as raw:
|
|
1642
|
+
repo = Path(raw)
|
|
1643
|
+
write_text(
|
|
1644
|
+
repo / "openspec/changes/demo/design.md",
|
|
1645
|
+
"## Decisions\n\nD1 — Unrelated decision. Basis: fixture authority.\n",
|
|
1646
|
+
)
|
|
1647
|
+
returncode, message = covers_message(repo)
|
|
1648
|
+
if returncode == 0 or not message.startswith(
|
|
1649
|
+
"Missing Covers critical statement: D2."
|
|
1650
|
+
):
|
|
1651
|
+
report(
|
|
1652
|
+
"unparsed-covers-critical-statement: an identifier absent from "
|
|
1653
|
+
f"design.md must still report Missing, got: {message!r}"
|
|
1654
|
+
)
|
|
1655
|
+
return 1
|
|
1656
|
+
|
|
1657
|
+
with tempfile.TemporaryDirectory(prefix="keel-unparsed-covers-unparsed-") as raw:
|
|
1658
|
+
repo = Path(raw)
|
|
1659
|
+
write_text(
|
|
1660
|
+
repo / "openspec/changes/demo/design.md",
|
|
1661
|
+
"## Decisions\n\n"
|
|
1662
|
+
"- **D2** — Keep one shared parser. Basis: fixture authority.\n",
|
|
1663
|
+
)
|
|
1664
|
+
returncode, message = covers_message(repo)
|
|
1665
|
+
if returncode == 0:
|
|
1666
|
+
report(
|
|
1667
|
+
"unparsed-covers-critical-statement: a present-but-mis-shaped D2 "
|
|
1668
|
+
"must still fail task-start."
|
|
1669
|
+
)
|
|
1670
|
+
return 1
|
|
1671
|
+
if not message.startswith("Unparsed Covers critical statement: D2."):
|
|
1672
|
+
report(
|
|
1673
|
+
"unparsed-covers-critical-statement: a present-but-mis-shaped D2 "
|
|
1674
|
+
f"must report Unparsed, got: {message!r}"
|
|
1675
|
+
)
|
|
1676
|
+
return 1
|
|
1677
|
+
if "D2 — one-line statement" not in message:
|
|
1678
|
+
report(
|
|
1679
|
+
"unparsed-covers-critical-statement: the Unparsed message must "
|
|
1680
|
+
f"name the required shape, got: {message!r}"
|
|
1681
|
+
)
|
|
1682
|
+
return 1
|
|
1683
|
+
|
|
1684
|
+
with tempfile.TemporaryDirectory(prefix="keel-unparsed-covers-resolved-") as raw:
|
|
1685
|
+
repo = Path(raw)
|
|
1686
|
+
write_text(
|
|
1687
|
+
repo / "openspec/changes/demo/design.md",
|
|
1688
|
+
"## Decisions\n\nD2 — Keep one shared parser. Basis: fixture authority.\n",
|
|
1689
|
+
)
|
|
1690
|
+
write_text(repo / "openspec/changes/demo/tasks.md", task)
|
|
1691
|
+
result = run_keel(
|
|
1692
|
+
repo, "gate", "task-start", "--change", "demo", "--task", "1.1", "--json"
|
|
1693
|
+
)
|
|
1694
|
+
if result.returncode != 0:
|
|
1695
|
+
report(
|
|
1696
|
+
"unparsed-covers-critical-statement: a correctly-shaped D2 must "
|
|
1697
|
+
f"still resolve. {(result.stderr or result.stdout).strip()}"
|
|
1698
|
+
)
|
|
1699
|
+
return 1
|
|
1700
|
+
authority = (
|
|
1701
|
+
json.loads(result.stdout)
|
|
1702
|
+
.get("contract", {})
|
|
1703
|
+
.get("capsule", {})
|
|
1704
|
+
.get("authority", [])
|
|
1705
|
+
)
|
|
1706
|
+
if not any(
|
|
1707
|
+
item.get("reference") == "D2" and item.get("kind") == "critical-statement"
|
|
1708
|
+
for item in authority
|
|
1709
|
+
):
|
|
1710
|
+
report(
|
|
1711
|
+
"unparsed-covers-critical-statement: correctly-shaped D2 did not "
|
|
1712
|
+
"resolve as critical-statement authority."
|
|
1713
|
+
)
|
|
1714
|
+
return 1
|
|
1715
|
+
|
|
1716
|
+
if "unparsed-covers-critical-statement" not in {name for name, _ in SCENARIOS}:
|
|
1717
|
+
report(
|
|
1718
|
+
"unparsed-covers-critical-statement: the scenario registry does not "
|
|
1719
|
+
"include it."
|
|
1720
|
+
)
|
|
1721
|
+
return 1
|
|
1722
|
+
report("unparsed-covers-critical-statement scenario passed.")
|
|
1723
|
+
return 0
|
|
1724
|
+
|
|
1725
|
+
|
|
1614
1726
|
def validate_expectation_completion_gates_scenario() -> int:
|
|
1615
1727
|
protocol_snippets = [
|
|
1616
1728
|
"Completion Gate",
|
|
@@ -2147,6 +2259,256 @@ def assert_target_overlays(
|
|
|
2147
2259
|
return None
|
|
2148
2260
|
|
|
2149
2261
|
|
|
2262
|
+
OVERLAY_ACTION_SKILLS = {
|
|
2263
|
+
"propose": "openspec-propose",
|
|
2264
|
+
"apply": "openspec-apply-change",
|
|
2265
|
+
"archive": "openspec-archive-change",
|
|
2266
|
+
"sync": "openspec-sync-specs",
|
|
2267
|
+
}
|
|
2268
|
+
|
|
2269
|
+
|
|
2270
|
+
def expected_overlay_surfaces(
|
|
2271
|
+
repo: Path,
|
|
2272
|
+
target: str,
|
|
2273
|
+
codex_home: Path | None = None,
|
|
2274
|
+
) -> list[Path]:
|
|
2275
|
+
"""Every surface `openspecOverlaySurfacesForTarget` projects the overlay onto.
|
|
2276
|
+
|
|
2277
|
+
Written out rather than discovered, so that a scan of the tree can be
|
|
2278
|
+
compared against it. A scan alone answers "the files that carry the
|
|
2279
|
+
marker", which is the same answer whether the CLI covered eight surfaces
|
|
2280
|
+
or none — and "none of them still carry it" is exactly what a broken
|
|
2281
|
+
surface list also reports.
|
|
2282
|
+
"""
|
|
2283
|
+
surfaces: list[Path] = []
|
|
2284
|
+
for action, skill in OVERLAY_ACTION_SKILLS.items():
|
|
2285
|
+
if action == "propose" and target == "opencode":
|
|
2286
|
+
continue
|
|
2287
|
+
if target == "claude":
|
|
2288
|
+
surfaces.append(repo / f".claude/skills/{skill}/SKILL.md")
|
|
2289
|
+
surfaces.append(repo / f".claude/commands/opsx/{action}.md")
|
|
2290
|
+
elif target == "codex":
|
|
2291
|
+
assert codex_home is not None
|
|
2292
|
+
surfaces.append(repo / f".codex/skills/{skill}/SKILL.md")
|
|
2293
|
+
surfaces.append(codex_home / f"prompts/opsx-{action}.md")
|
|
2294
|
+
else:
|
|
2295
|
+
surfaces.append(repo / f".opencode/skills/{skill}/SKILL.md")
|
|
2296
|
+
surfaces.append(repo / f".opencode/commands/opsx-{action}.md")
|
|
2297
|
+
return surfaces
|
|
2298
|
+
|
|
2299
|
+
|
|
2300
|
+
def files_carrying_overlay(*roots: Path) -> set[Path]:
|
|
2301
|
+
found: set[Path] = set()
|
|
2302
|
+
for root in roots:
|
|
2303
|
+
if not root.is_dir():
|
|
2304
|
+
continue
|
|
2305
|
+
for path in root.rglob("*.md"):
|
|
2306
|
+
if not path.is_file():
|
|
2307
|
+
continue
|
|
2308
|
+
text = path.read_text(encoding="utf-8", errors="replace")
|
|
2309
|
+
if "keel:openspec-surface-overlay" in text:
|
|
2310
|
+
found.add(path)
|
|
2311
|
+
return found
|
|
2312
|
+
|
|
2313
|
+
|
|
2314
|
+
def validate_uninstall_removes_the_overlay_scenario() -> int:
|
|
2315
|
+
"""Uninstalling left Keel's instructions in files OpenSpec owns.
|
|
2316
|
+
|
|
2317
|
+
The overlay block is written into the `opsx` command surfaces and the
|
|
2318
|
+
`openspec-*` skills, and every line of it is about Keel: the gate to run,
|
|
2319
|
+
the checklist to run, how to invoke OpenSpec, where standing authorization
|
|
2320
|
+
is declared. `keel --uninstall` removed the `keel:start` managed block from
|
|
2321
|
+
AGENTS.md and CLAUDE.md and left all of that behind.
|
|
2322
|
+
|
|
2323
|
+
Measured 2026-08-03 at 5.22.0: Claude 8 of 8 surfaces, Codex 8 of 8 (four
|
|
2324
|
+
of them under `CODEX_HOME/prompts/`), OpenCode 6 of 6 still carried the
|
|
2325
|
+
marker after uninstall. Filed as
|
|
2326
|
+
https://github.com/TanglmChris/keel/issues/50.
|
|
2327
|
+
"""
|
|
2328
|
+
label = "uninstall-removes-the-overlay"
|
|
2329
|
+
with tempfile.TemporaryDirectory(prefix="keel-uninstall-overlay-") as raw:
|
|
2330
|
+
tmp = Path(raw)
|
|
2331
|
+
|
|
2332
|
+
for target in ("claude", "codex", "opencode"):
|
|
2333
|
+
repo = tmp / target
|
|
2334
|
+
repo.mkdir()
|
|
2335
|
+
env = None
|
|
2336
|
+
codex_home = None
|
|
2337
|
+
roots = [repo]
|
|
2338
|
+
if target == "codex":
|
|
2339
|
+
codex_home = tmp / "codex-home"
|
|
2340
|
+
env = os.environ.copy()
|
|
2341
|
+
env["CODEX_HOME"] = str(codex_home)
|
|
2342
|
+
roots.append(codex_home)
|
|
2343
|
+
|
|
2344
|
+
init = run_keel(repo, "--init", "--target", target, env=env)
|
|
2345
|
+
if init.returncode != 0:
|
|
2346
|
+
report(f"{label} `keel --init --target {target}` failed.")
|
|
2347
|
+
report((init.stderr or init.stdout).strip())
|
|
2348
|
+
return 1
|
|
2349
|
+
|
|
2350
|
+
# M1 positive control — the surfaces the CLI writes to are the ones
|
|
2351
|
+
# this scenario is about to check, asserted before the act. Without
|
|
2352
|
+
# it, a run whose surface list resolved to nothing reports the same
|
|
2353
|
+
# clean tree as a run that removed everything.
|
|
2354
|
+
expected = set(expected_overlay_surfaces(repo, target, codex_home))
|
|
2355
|
+
carrying = files_carrying_overlay(*roots)
|
|
2356
|
+
if carrying != expected:
|
|
2357
|
+
report(
|
|
2358
|
+
f"{label} M1 the {target} surfaces carrying the overlay "
|
|
2359
|
+
"after init are not the ones this scenario expects, so "
|
|
2360
|
+
"nothing after this measures removal."
|
|
2361
|
+
)
|
|
2362
|
+
report(f" only installed: {sorted(str(p) for p in carrying - expected)}")
|
|
2363
|
+
report(f" only expected: {sorted(str(p) for p in expected - carrying)}")
|
|
2364
|
+
return 1
|
|
2365
|
+
|
|
2366
|
+
# M2 — what the file held before Keel's block, kept so the whole
|
|
2367
|
+
# file can be compared against it afterwards. Trailing newlines are
|
|
2368
|
+
# normalized to one because the separator the install side inserted
|
|
2369
|
+
# before the block goes with the block.
|
|
2370
|
+
bodies = {}
|
|
2371
|
+
for surface in expected:
|
|
2372
|
+
text = surface.read_text(encoding="utf-8")
|
|
2373
|
+
head = text[: text.index("<!-- keel:openspec-surface-overlay")]
|
|
2374
|
+
bodies[surface] = head.rstrip("\n") + "\n"
|
|
2375
|
+
|
|
2376
|
+
uninstall = run_keel(repo, "--uninstall", "--target", target, env=env)
|
|
2377
|
+
if uninstall.returncode != 0:
|
|
2378
|
+
report(f"{label} M1 `keel --uninstall --target {target}` failed.")
|
|
2379
|
+
report((uninstall.stderr or uninstall.stdout).strip())
|
|
2380
|
+
return 1
|
|
2381
|
+
|
|
2382
|
+
# M1 — nothing Keel wrote is left in a file OpenSpec owns.
|
|
2383
|
+
left = files_carrying_overlay(*roots)
|
|
2384
|
+
if left:
|
|
2385
|
+
report(
|
|
2386
|
+
f"{label} M1 uninstalling {target} left the Keel overlay "
|
|
2387
|
+
f"in {len(left)} of {len(expected)} surfaces, so the "
|
|
2388
|
+
"uninstalled repository still instructs an agent to run "
|
|
2389
|
+
"commands that were just removed."
|
|
2390
|
+
)
|
|
2391
|
+
for path in sorted(str(p) for p in left):
|
|
2392
|
+
report(f" {path}")
|
|
2393
|
+
return 1
|
|
2394
|
+
|
|
2395
|
+
# M4 — the run says how many surfaces it cleaned. An uninstall that
|
|
2396
|
+
# silently does nothing and an uninstall that cleaned everything
|
|
2397
|
+
# produce the same tree on a repository that was never installed.
|
|
2398
|
+
if f"removed={len(expected)}" not in uninstall.stdout:
|
|
2399
|
+
report(
|
|
2400
|
+
f"{label} M4 uninstalling {target} did not report the "
|
|
2401
|
+
f"{len(expected)} overlays it removed."
|
|
2402
|
+
)
|
|
2403
|
+
report((uninstall.stderr or uninstall.stdout).strip())
|
|
2404
|
+
return 1
|
|
2405
|
+
|
|
2406
|
+
# M2 — the block, and only the block. The file is OpenSpec's, so a
|
|
2407
|
+
# removal that took the marker and anything either side of it is a
|
|
2408
|
+
# worse defect than the one being fixed, and "the marker is gone"
|
|
2409
|
+
# cannot tell the two apart.
|
|
2410
|
+
for surface, body in bodies.items():
|
|
2411
|
+
if not surface.is_file():
|
|
2412
|
+
report(
|
|
2413
|
+
f"{label} M2 uninstalling {target} deleted {surface}, "
|
|
2414
|
+
"which is OpenSpec's file. Removing Keel's block is "
|
|
2415
|
+
"the whole obligation."
|
|
2416
|
+
)
|
|
2417
|
+
return 1
|
|
2418
|
+
after = surface.read_text(encoding="utf-8")
|
|
2419
|
+
if after != body:
|
|
2420
|
+
report(
|
|
2421
|
+
f"{label} M2 uninstalling {target} did not leave "
|
|
2422
|
+
f"{surface} at the bytes that preceded the overlay."
|
|
2423
|
+
)
|
|
2424
|
+
report(f" expected {len(body)} bytes ending {body[-40:]!r}")
|
|
2425
|
+
report(f" found {len(after)} bytes ending {after[-40:]!r}")
|
|
2426
|
+
return 1
|
|
2427
|
+
|
|
2428
|
+
# M3 — a dry run plans the writes it would make. `--check` already had
|
|
2429
|
+
# to learn that a Node-side step the installer's plan cannot see makes
|
|
2430
|
+
# a dry run under-report a run that writes; the uninstall side has the
|
|
2431
|
+
# same shape.
|
|
2432
|
+
dry = tmp / "dry-run"
|
|
2433
|
+
dry.mkdir()
|
|
2434
|
+
dry_init = run_keel(dry, "--init", "--target", "claude")
|
|
2435
|
+
if dry_init.returncode != 0:
|
|
2436
|
+
report(f"{label} M3 dry-run fixture init failed.")
|
|
2437
|
+
report((dry_init.stderr or dry_init.stdout).strip())
|
|
2438
|
+
return 1
|
|
2439
|
+
dry_expected = set(expected_overlay_surfaces(dry, "claude"))
|
|
2440
|
+
dry_run = run_keel(dry, "--uninstall", "--target", "claude", "--dry-run")
|
|
2441
|
+
if dry_run.returncode != 0:
|
|
2442
|
+
report(f"{label} M3 `keel --uninstall --dry-run` failed.")
|
|
2443
|
+
report((dry_run.stderr or dry_run.stdout).strip())
|
|
2444
|
+
return 1
|
|
2445
|
+
unplanned = [
|
|
2446
|
+
str(surface)
|
|
2447
|
+
for surface in sorted(dry_expected)
|
|
2448
|
+
if str(surface) not in dry_run.stdout
|
|
2449
|
+
]
|
|
2450
|
+
if unplanned:
|
|
2451
|
+
report(
|
|
2452
|
+
f"{label} M3 the dry run did not name {len(unplanned)} of "
|
|
2453
|
+
f"{len(dry_expected)} surfaces it would clean, so the plan "
|
|
2454
|
+
"under-reports a run that writes."
|
|
2455
|
+
)
|
|
2456
|
+
for surface in unplanned:
|
|
2457
|
+
report(f" {surface}")
|
|
2458
|
+
return 1
|
|
2459
|
+
if files_carrying_overlay(dry) != dry_expected:
|
|
2460
|
+
report(
|
|
2461
|
+
f"{label} M3 the dry run removed overlays. A dry run reports "
|
|
2462
|
+
"what it would do and writes nothing."
|
|
2463
|
+
)
|
|
2464
|
+
return 1
|
|
2465
|
+
|
|
2466
|
+
# M4 — uninstalling again, and uninstalling a repository that never
|
|
2467
|
+
# received the overlay, both succeed. Uninstall is something a user
|
|
2468
|
+
# reaches for when something is already wrong; failing on the second
|
|
2469
|
+
# attempt is the worst moment to fail.
|
|
2470
|
+
again = run_keel(tmp / "claude", "--uninstall", "--target", "claude")
|
|
2471
|
+
if again.returncode != 0:
|
|
2472
|
+
report(
|
|
2473
|
+
f"{label} M4 a second uninstall failed. Uninstall is reached "
|
|
2474
|
+
"when something is already wrong; failing on the second "
|
|
2475
|
+
"attempt is the worst moment to fail."
|
|
2476
|
+
)
|
|
2477
|
+
report((again.stderr or again.stdout).strip())
|
|
2478
|
+
return 1
|
|
2479
|
+
if "removed=0" not in again.stdout:
|
|
2480
|
+
report(
|
|
2481
|
+
f"{label} M4 a second uninstall did not report that it removed "
|
|
2482
|
+
"nothing, so a run that cleaned nothing reads like a run that "
|
|
2483
|
+
"cleaned everything."
|
|
2484
|
+
)
|
|
2485
|
+
report((again.stderr or again.stdout).strip())
|
|
2486
|
+
return 1
|
|
2487
|
+
|
|
2488
|
+
bare = tmp / "install-only"
|
|
2489
|
+
bare.mkdir()
|
|
2490
|
+
bare_install = run_keel(bare, "--install", "--target", "claude")
|
|
2491
|
+
if bare_install.returncode != 0:
|
|
2492
|
+
report(f"{label} M4 install-only fixture failed.")
|
|
2493
|
+
report((bare_install.stderr or bare_install.stdout).strip())
|
|
2494
|
+
return 1
|
|
2495
|
+
bare_uninstall = run_keel(bare, "--uninstall", "--target", "claude")
|
|
2496
|
+
if bare_uninstall.returncode != 0:
|
|
2497
|
+
report(
|
|
2498
|
+
f"{label} M4 uninstalling a repository whose OpenSpec surfaces "
|
|
2499
|
+
"were never created failed. An absent surface is nothing to "
|
|
2500
|
+
"remove, not an error."
|
|
2501
|
+
)
|
|
2502
|
+
report((bare_uninstall.stderr or bare_uninstall.stdout).strip())
|
|
2503
|
+
return 1
|
|
2504
|
+
|
|
2505
|
+
if label not in {name for name, _ in SCENARIOS}:
|
|
2506
|
+
report(f"{label}: the scenario registry does not include it.")
|
|
2507
|
+
return 1
|
|
2508
|
+
report(f"{label} scenario passed.")
|
|
2509
|
+
return 0
|
|
2510
|
+
|
|
2511
|
+
|
|
2150
2512
|
def validate_sync_surface_overlay_scenario() -> int:
|
|
2151
2513
|
"""The surface that performs a gated action never named its gate.
|
|
2152
2514
|
|
|
@@ -2253,11 +2615,11 @@ def validate_sync_surface_overlay_scenario() -> int:
|
|
|
2253
2615
|
)
|
|
2254
2616
|
return 1
|
|
2255
2617
|
|
|
2256
|
-
# Uninstall is
|
|
2257
|
-
#
|
|
2258
|
-
#
|
|
2259
|
-
#
|
|
2260
|
-
#
|
|
2618
|
+
# Uninstall is asserted by `uninstall-removes-the-overlay`, which
|
|
2619
|
+
# covers every surface on every target rather than sync alone. It
|
|
2620
|
+
# stripped nothing from any surface when this scenario was written;
|
|
2621
|
+
# that was #50, and the assertion lives there because the contract
|
|
2622
|
+
# is about all four actions, not about this one.
|
|
2261
2623
|
|
|
2262
2624
|
# M4 — covering the surface did not widen what may be standing-authorized.
|
|
2263
2625
|
config_doc = (ROOT / "keel/config.yaml").read_text(encoding="utf-8")
|
|
@@ -2276,6 +2638,19 @@ def validate_sync_surface_overlay_scenario() -> int:
|
|
|
2276
2638
|
return 0
|
|
2277
2639
|
|
|
2278
2640
|
|
|
2641
|
+
def overlay_action_label(text: str, pattern: str) -> str | None:
|
|
2642
|
+
"""The action list a summary line names, or None when the line is absent.
|
|
2643
|
+
|
|
2644
|
+
`\\S*` rather than `\\S+` on purpose: a label that derived to nothing leaves
|
|
2645
|
+
two spaces where the actions belong, and matching it is what lets an empty
|
|
2646
|
+
label be reported as empty instead of as a missing line.
|
|
2647
|
+
"""
|
|
2648
|
+
match = re.search(pattern, text)
|
|
2649
|
+
if match is None:
|
|
2650
|
+
return None
|
|
2651
|
+
return match.group(1)
|
|
2652
|
+
|
|
2653
|
+
|
|
2279
2654
|
def validate_openspec_surface_overlay_scenario() -> int:
|
|
2280
2655
|
with tempfile.TemporaryDirectory(prefix="keel-overlay-") as raw_tmp:
|
|
2281
2656
|
tmp = Path(raw_tmp)
|
|
@@ -2300,6 +2675,61 @@ def validate_openspec_surface_overlay_scenario() -> int:
|
|
|
2300
2675
|
report((claude_doctor.stderr or claude_doctor.stdout).strip())
|
|
2301
2676
|
return 1
|
|
2302
2677
|
|
|
2678
|
+
# Every direction that reports the overlay names the actions it covers,
|
|
2679
|
+
# and all of them describe one surface list. They are compared against
|
|
2680
|
+
# each other rather than against a literal repeated here: the doctor
|
|
2681
|
+
# label is already pinned above, so an action joining the managed set
|
|
2682
|
+
# moves all three lines together or fails on this comparison. A third
|
|
2683
|
+
# copy of the string is what produced the defect this guards.
|
|
2684
|
+
#
|
|
2685
|
+
# The refresh line was the copy nobody was watching. Measured 2026-08-03
|
|
2686
|
+
# on one repository, back to back: `--init` reported `apply/archive`
|
|
2687
|
+
# while `--uninstall` reported `apply/archive/sync` (issue #75).
|
|
2688
|
+
#
|
|
2689
|
+
# Absence is checked before agreement, and separately, because two
|
|
2690
|
+
# labels that were never found also agree.
|
|
2691
|
+
claude_uninstall = run_keel(claude_repo, "--uninstall", "--target", "claude")
|
|
2692
|
+
if claude_uninstall.returncode != 0:
|
|
2693
|
+
report("openspec-surface-overlay scenario Claude uninstall failed:")
|
|
2694
|
+
report((claude_uninstall.stderr or claude_uninstall.stdout).strip())
|
|
2695
|
+
return 1
|
|
2696
|
+
summaries = {
|
|
2697
|
+
"refresh": overlay_action_label(
|
|
2698
|
+
claude_init.stdout, r"OpenSpec (\S*) overlay refreshed="
|
|
2699
|
+
),
|
|
2700
|
+
"doctor": overlay_action_label(
|
|
2701
|
+
claude_doctor.stdout, r"Keel (\S*) overlay:"
|
|
2702
|
+
),
|
|
2703
|
+
"removal": overlay_action_label(
|
|
2704
|
+
claude_uninstall.stdout, r"OpenSpec (\S*) overlay removed="
|
|
2705
|
+
),
|
|
2706
|
+
}
|
|
2707
|
+
for direction, printed in summaries.items():
|
|
2708
|
+
if printed is None:
|
|
2709
|
+
report(
|
|
2710
|
+
f"openspec-surface-overlay scenario found no {direction} "
|
|
2711
|
+
"overlay summary at all, so there was no label to compare "
|
|
2712
|
+
"rather than a label that disagreed."
|
|
2713
|
+
)
|
|
2714
|
+
return 1
|
|
2715
|
+
if not printed:
|
|
2716
|
+
report(
|
|
2717
|
+
f"openspec-surface-overlay scenario read an empty action "
|
|
2718
|
+
f"label from the {direction} summary; a label that derives "
|
|
2719
|
+
"to nothing reports no actions at all."
|
|
2720
|
+
)
|
|
2721
|
+
return 1
|
|
2722
|
+
for direction in ("refresh", "removal"):
|
|
2723
|
+
if summaries[direction] != summaries["doctor"]:
|
|
2724
|
+
report(
|
|
2725
|
+
"openspec-surface-overlay scenario overlay summaries "
|
|
2726
|
+
f"disagree: {direction} names {summaries[direction]!r} and "
|
|
2727
|
+
f"doctor names {summaries['doctor']!r}. They describe the "
|
|
2728
|
+
"same managed surface list, so one of them is written "
|
|
2729
|
+
"beside the action set instead of derived from it."
|
|
2730
|
+
)
|
|
2731
|
+
return 1
|
|
2732
|
+
|
|
2303
2733
|
codex_repo = tmp / "codex"
|
|
2304
2734
|
codex_repo.mkdir()
|
|
2305
2735
|
codex_home = tmp / "codex-home"
|
|
@@ -2762,6 +3192,21 @@ def validate_cli_scenario() -> int:
|
|
|
2762
3192
|
report((help_result.stderr or help_result.stdout).strip())
|
|
2763
3193
|
return 1
|
|
2764
3194
|
|
|
3195
|
+
usage_block = help_result.stdout.split("Usage:", 1)[1].split("\n\n", 1)[0]
|
|
3196
|
+
triage_lines = [line for line in usage_block.splitlines() if "keel triage" in line]
|
|
3197
|
+
if not triage_lines:
|
|
3198
|
+
report("cli scenario expected keel --help Usage: block to list a `keel triage` line.")
|
|
3199
|
+
report(usage_block.strip())
|
|
3200
|
+
return 1
|
|
3201
|
+
if "--labels" not in triage_lines[0]:
|
|
3202
|
+
report("cli scenario expected the keel --help `keel triage` line to name --labels.")
|
|
3203
|
+
report(triage_lines[0].strip())
|
|
3204
|
+
return 1
|
|
3205
|
+
if "--issue" not in triage_lines[0]:
|
|
3206
|
+
report("cli scenario expected the keel --help `keel triage` line to name --issue.")
|
|
3207
|
+
report(triage_lines[0].strip())
|
|
3208
|
+
return 1
|
|
3209
|
+
|
|
2765
3210
|
version_result = run_keel(ROOT, "--version")
|
|
2766
3211
|
expected_version = f"keel {PACKAGE_VERSION}"
|
|
2767
3212
|
if version_result.returncode != 0 or version_result.stdout.strip() != expected_version:
|
|
@@ -3837,55 +4282,368 @@ def validate_task_start_invalidation_scenario() -> int:
|
|
|
3837
4282
|
return 0
|
|
3838
4283
|
|
|
3839
4284
|
|
|
3840
|
-
|
|
3841
|
-
|
|
3842
|
-
|
|
3843
|
-
|
|
3844
|
-
|
|
3845
|
-
|
|
3846
|
-
|
|
3847
|
-
|
|
3848
|
-
|
|
3849
|
-
)
|
|
4285
|
+
# A task for the section-boundary fixtures. Two shapes matter and nothing else
|
|
4286
|
+
# does: `covers` puts `- E<n>:` lines inside a task body, which a section that
|
|
4287
|
+
# ran over the task list read as its own entries, and `repo_action` gives the
|
|
4288
|
+
# task a `Touch` of a bare `- none`, which such a section read as its `- None.`
|
|
4289
|
+
# and closed itself with. Every other field is the cheapest legitimate value.
|
|
4290
|
+
def section_boundary_task(
|
|
4291
|
+
task_id: str,
|
|
4292
|
+
*,
|
|
4293
|
+
checked: bool,
|
|
4294
|
+
covers: tuple[str, ...] = ("E1: the task owns its file",),
|
|
4295
|
+
repo_action: bool = False,
|
|
4296
|
+
) -> str:
|
|
4297
|
+
covers_lines = "".join(f" - {item}\n" for item in covers)
|
|
4298
|
+
touch = " - none\n" if repo_action else " - src/example.js\n"
|
|
4299
|
+
mode = "repo-action" if repo_action else "implementation"
|
|
4300
|
+
return (
|
|
4301
|
+
f"- [{'x' if checked else ' '}] {task_id} Section boundary fixture\n"
|
|
4302
|
+
" - Owner: claude\n"
|
|
4303
|
+
f" - Mode: {mode}\n"
|
|
4304
|
+
" - Covers:\n"
|
|
4305
|
+
f"{covers_lines}"
|
|
4306
|
+
" - Read:\n"
|
|
4307
|
+
" - README.md\n"
|
|
4308
|
+
" - Touch:\n"
|
|
4309
|
+
f"{touch}"
|
|
4310
|
+
" - Verify:\n"
|
|
4311
|
+
" - Strategy: evidence-first\n"
|
|
4312
|
+
" - M1: node test.js reports the public behavior passing\n"
|
|
4313
|
+
" - Acceptance:\n"
|
|
4314
|
+
" - Public behavior passes.\n"
|
|
4315
|
+
" - Autonomy boundary:\n"
|
|
4316
|
+
" - Default: hard-stop\n"
|
|
4317
|
+
" - Pre-authorized fallback: none\n"
|
|
4318
|
+
" - Stop Rules:\n"
|
|
4319
|
+
" - Stop on failure.\n"
|
|
4320
|
+
" - Evidence:\n"
|
|
4321
|
+
" - Contract: pending\n"
|
|
4322
|
+
" - M1: node test.js reported 4 passing and 0 failing\n"
|
|
4323
|
+
" - Review:\n"
|
|
4324
|
+
" - Status: pass\n"
|
|
4325
|
+
" - Acceptance check: the public behavior was exercised\n"
|
|
4326
|
+
" - Scope check: only the declared file changed\n"
|
|
4327
|
+
" - Findings: none\n"
|
|
4328
|
+
" - Blocker: none\n"
|
|
4329
|
+
" - Stop if:\n"
|
|
4330
|
+
" - Requires files outside Touch.\n"
|
|
4331
|
+
)
|
|
3850
4332
|
|
|
3851
4333
|
|
|
3852
|
-
|
|
3853
|
-
|
|
4334
|
+
# The same file with the section above the task list and in the tail. Nothing
|
|
4335
|
+
# else differs between the two, which is what makes the pair a measurement of
|
|
4336
|
+
# position rather than of content.
|
|
4337
|
+
def section_boundary_tasks_md(*, section: str, task: str, position: str) -> str:
|
|
4338
|
+
if position == "above":
|
|
4339
|
+
return "# Tasks\n\n" + section + "\n" + task
|
|
4340
|
+
return "# Tasks\n\n" + task + "\n" + section
|
|
3854
4341
|
|
|
3855
|
-
# A dry run is relied on, so it is wrong in both directions: naming a write
|
|
3856
|
-
# that will not happen trains the reader to ignore it, and omitting one
|
|
3857
|
-
# breaks the promise the dry run exists to make. `--check` used to omit the
|
|
3858
|
-
# overlay step entirely while `--install --dry-run` claimed every surface.
|
|
3859
|
-
def overlay_lines(text: str) -> list[str]:
|
|
3860
|
-
return [line for line in text.splitlines() if "overlay" in line]
|
|
3861
4342
|
|
|
3862
|
-
|
|
3863
|
-
|
|
3864
|
-
|
|
4343
|
+
def section_boundary_repo(root: Path, name: str, content: str) -> Path:
|
|
4344
|
+
repo = root / name
|
|
4345
|
+
change = repo / "openspec/changes/demo"
|
|
4346
|
+
write_text(change / "tasks.md", content)
|
|
4347
|
+
write_text(change / "proposal.md", "# Proposal\n")
|
|
4348
|
+
write_text(change / "design.md", "## Context\n\nfixture\n")
|
|
4349
|
+
write_text(change / "specs/demo/spec.md", "## ADDED Requirements\n")
|
|
4350
|
+
return repo
|
|
3865
4351
|
|
|
3866
|
-
with tempfile.TemporaryDirectory(prefix="keel-dry-run-overlay-") as raw_tmp:
|
|
3867
|
-
repo = Path(raw_tmp) / "repo"
|
|
3868
|
-
repo.mkdir()
|
|
3869
|
-
# The overlay surfaces are files OpenSpec generates; install merges into
|
|
3870
|
-
# them and skips the ones that are absent. Create them so this scenario
|
|
3871
|
-
# exercises the classification rather than the missing branch.
|
|
3872
|
-
for relative in (
|
|
3873
|
-
".claude/skills/openspec-propose/SKILL.md",
|
|
3874
|
-
".claude/skills/openspec-apply-change/SKILL.md",
|
|
3875
|
-
".claude/skills/openspec-archive-change/SKILL.md",
|
|
3876
|
-
".claude/commands/opsx/propose.md",
|
|
3877
|
-
".claude/commands/opsx/apply.md",
|
|
3878
|
-
".claude/commands/opsx/archive.md",
|
|
3879
|
-
):
|
|
3880
|
-
write_text(repo / relative, "# OpenSpec surface\n\nGenerated body.\n")
|
|
3881
|
-
if run_keel(repo, "--install", "--target", "claude").returncode != 0:
|
|
3882
|
-
report(f"{label} could not install a fixture repository.")
|
|
3883
|
-
return 1
|
|
3884
4352
|
|
|
3885
|
-
|
|
3886
|
-
|
|
3887
|
-
|
|
3888
|
-
|
|
4353
|
+
def validate_section_boundary_scenario() -> int:
|
|
4354
|
+
"""A change-level section is bounded by the tasks, not only by headings.
|
|
4355
|
+
|
|
4356
|
+
Regression for issue #71 and for the silent half the issue does not report.
|
|
4357
|
+
Both readers of a change-level section sliced to the next `## ` heading, so
|
|
4358
|
+
a section that was not the file's last one ran over the task list: the
|
|
4359
|
+
`- E<n>:` lines a task declares under `Covers` were judged as coverage
|
|
4360
|
+
entries and reported unclosed, and a `repo-action` task's `Touch` of a bare
|
|
4361
|
+
`- none` was read as the section's `- None.` and closed a declaration that
|
|
4362
|
+
closed nothing.
|
|
4363
|
+
|
|
4364
|
+
Every cell is a pair — the identical section above the task list and in the
|
|
4365
|
+
tail — asserted both absolutely and against each other. The absolute half
|
|
4366
|
+
matters: a comparison alone passes when both positions are broken the same
|
|
4367
|
+
way.
|
|
4368
|
+
"""
|
|
4369
|
+
label = "section-boundary"
|
|
4370
|
+
|
|
4371
|
+
def problems_of(payload: dict) -> list[tuple[str, str]]:
|
|
4372
|
+
return sorted(
|
|
4373
|
+
(item.get("code", ""), item.get("message", ""))
|
|
4374
|
+
for item in payload.get("problems", [])
|
|
4375
|
+
)
|
|
4376
|
+
|
|
4377
|
+
def closure_problems(payload: dict, code: str) -> list[str]:
|
|
4378
|
+
return [
|
|
4379
|
+
item.get("message", "")
|
|
4380
|
+
for item in payload.get("problems", [])
|
|
4381
|
+
if item.get("code") == code
|
|
4382
|
+
]
|
|
4383
|
+
|
|
4384
|
+
with tempfile.TemporaryDirectory(
|
|
4385
|
+
prefix="keel-section-boundary-", ignore_cleanup_errors=True
|
|
4386
|
+
) as raw:
|
|
4387
|
+
tmp = Path(raw)
|
|
4388
|
+
|
|
4389
|
+
# `None` and `{}` are kept apart on purpose. A fixture task-start
|
|
4390
|
+
# refused and a gate that printed nothing are different failures, and a
|
|
4391
|
+
# message naming one of them while the other happened sends the reader
|
|
4392
|
+
# to a place with no problem in it.
|
|
4393
|
+
def close(name: str, content: str) -> dict | None:
|
|
4394
|
+
repo = section_boundary_repo(tmp, name, content)
|
|
4395
|
+
# A checked task with no recorded anchor fails the close for a
|
|
4396
|
+
# reason that has nothing to do with this boundary, and that noise
|
|
4397
|
+
# would land in both halves of every pair.
|
|
4398
|
+
if not record_contract_anchor(repo, "demo", "1.1"):
|
|
4399
|
+
return None
|
|
4400
|
+
result = run_keel(
|
|
4401
|
+
repo, "gate", "change-close", ".",
|
|
4402
|
+
"--change", "demo", "--action", "archive", "--json",
|
|
4403
|
+
)
|
|
4404
|
+
return json.loads(result.stdout) if result.stdout.strip() else {}
|
|
4405
|
+
|
|
4406
|
+
def start(name: str, content: str) -> dict:
|
|
4407
|
+
repo = section_boundary_repo(tmp, name, content)
|
|
4408
|
+
result = run_keel(
|
|
4409
|
+
repo, "gate", "task-start", ".",
|
|
4410
|
+
"--change", "demo", "--task", "1.1", "--no-guard", "--json",
|
|
4411
|
+
)
|
|
4412
|
+
return json.loads(result.stdout) if result.stdout.strip() else {}
|
|
4413
|
+
|
|
4414
|
+
def readable(payload: dict | None, fixture: str, gate: str) -> bool:
|
|
4415
|
+
if payload is None:
|
|
4416
|
+
report(
|
|
4417
|
+
f"{label} could not record a Contract anchor for {fixture}: "
|
|
4418
|
+
"task-start refused the fixture, so change-close never ran "
|
|
4419
|
+
"against it."
|
|
4420
|
+
)
|
|
4421
|
+
return False
|
|
4422
|
+
if not payload:
|
|
4423
|
+
report(
|
|
4424
|
+
f"{label} got no JSON from {gate} for {fixture}, so there "
|
|
4425
|
+
"was no verdict to read rather than a verdict that was "
|
|
4426
|
+
"wrong."
|
|
4427
|
+
)
|
|
4428
|
+
return False
|
|
4429
|
+
return True
|
|
4430
|
+
|
|
4431
|
+
# Cell 1: `## Expectation Coverage` closes every entry it declares, and
|
|
4432
|
+
# the task below it declares `- E1:`/`- E2:` under its own `Covers`.
|
|
4433
|
+
# Nothing here is unclosed in either position.
|
|
4434
|
+
closed_coverage = (
|
|
4435
|
+
"## Invalidates\n\n- None.\n\n"
|
|
4436
|
+
"## Expectation Coverage\n\n"
|
|
4437
|
+
"- E1: The task owns its file. Covered by: 1.1\n"
|
|
4438
|
+
"- E2: The task proves it. Covered by: 1.1\n"
|
|
4439
|
+
)
|
|
4440
|
+
closed_task = section_boundary_task(
|
|
4441
|
+
"1.1",
|
|
4442
|
+
checked=True,
|
|
4443
|
+
covers=("E1: The task owns its file.", "E2: The task proves it."),
|
|
4444
|
+
)
|
|
4445
|
+
closed = {
|
|
4446
|
+
position: close(
|
|
4447
|
+
f"coverage-closed-{position}",
|
|
4448
|
+
section_boundary_tasks_md(
|
|
4449
|
+
section=closed_coverage, task=closed_task, position=position
|
|
4450
|
+
),
|
|
4451
|
+
)
|
|
4452
|
+
for position in ("above", "tail")
|
|
4453
|
+
}
|
|
4454
|
+
for position, payload in closed.items():
|
|
4455
|
+
if not readable(payload, f"coverage-closed-{position}", "change-close"):
|
|
4456
|
+
return 1
|
|
4457
|
+
stray = closure_problems(payload, "expectation-closure")
|
|
4458
|
+
if stray:
|
|
4459
|
+
report(
|
|
4460
|
+
f"{label} reported an expectation the section closes as "
|
|
4461
|
+
f"unclosed, with the section {position} the task list. The "
|
|
4462
|
+
"entries it judged are the task's own Covers lines."
|
|
4463
|
+
)
|
|
4464
|
+
report(repr(stray))
|
|
4465
|
+
return 1
|
|
4466
|
+
if problems_of(closed["above"]) != problems_of(closed["tail"]):
|
|
4467
|
+
report(
|
|
4468
|
+
f"{label} returned different problems for identical section "
|
|
4469
|
+
"content depending on where the section sits."
|
|
4470
|
+
)
|
|
4471
|
+
report(repr(problems_of(closed["above"])))
|
|
4472
|
+
report(repr(problems_of(closed["tail"])))
|
|
4473
|
+
return 1
|
|
4474
|
+
if closed["tail"].get("status") != "pass":
|
|
4475
|
+
report(
|
|
4476
|
+
f"{label} refused a fixture whose only expectations are closed; "
|
|
4477
|
+
"the pair comparison above would then be comparing two failures."
|
|
4478
|
+
)
|
|
4479
|
+
report(repr(closed["tail"].get("problems")))
|
|
4480
|
+
return 1
|
|
4481
|
+
|
|
4482
|
+
# Cell 2: the silent half. The entry closes nothing, and the task below
|
|
4483
|
+
# is a `repo-action` whose `Touch` is a bare `- none`.
|
|
4484
|
+
open_coverage = (
|
|
4485
|
+
"## Invalidates\n\n- None.\n\n"
|
|
4486
|
+
"## Expectation Coverage\n\n"
|
|
4487
|
+
"- E3: Nothing closes this one.\n"
|
|
4488
|
+
)
|
|
4489
|
+
none_task = section_boundary_task("1.1", checked=True, repo_action=True)
|
|
4490
|
+
unclosed = {
|
|
4491
|
+
position: close(
|
|
4492
|
+
f"coverage-open-{position}",
|
|
4493
|
+
section_boundary_tasks_md(
|
|
4494
|
+
section=open_coverage, task=none_task, position=position
|
|
4495
|
+
),
|
|
4496
|
+
)
|
|
4497
|
+
for position in ("above", "tail")
|
|
4498
|
+
}
|
|
4499
|
+
for position, payload in unclosed.items():
|
|
4500
|
+
if not readable(payload, f"coverage-open-{position}", "change-close"):
|
|
4501
|
+
return 1
|
|
4502
|
+
named = [
|
|
4503
|
+
message
|
|
4504
|
+
for message in closure_problems(payload, "expectation-closure")
|
|
4505
|
+
if "E3" in message
|
|
4506
|
+
]
|
|
4507
|
+
if not named:
|
|
4508
|
+
report(
|
|
4509
|
+
f"{label} accepted an expectation that closes nothing, with "
|
|
4510
|
+
f"the section {position} the task list. A task field of "
|
|
4511
|
+
"`none` is not the section's `- None.`."
|
|
4512
|
+
)
|
|
4513
|
+
report(repr(payload.get("problems")))
|
|
4514
|
+
return 1
|
|
4515
|
+
if problems_of(unclosed["above"]) != problems_of(unclosed["tail"]):
|
|
4516
|
+
report(
|
|
4517
|
+
f"{label} returned different problems for an identical unclosed "
|
|
4518
|
+
"section depending on where the section sits."
|
|
4519
|
+
)
|
|
4520
|
+
report(repr(problems_of(unclosed["above"])))
|
|
4521
|
+
report(repr(problems_of(unclosed["tail"])))
|
|
4522
|
+
return 1
|
|
4523
|
+
|
|
4524
|
+
# Cell 3: the other reader. `## Invalidates` is read by task-start, and
|
|
4525
|
+
# holds the character-identical slice.
|
|
4526
|
+
closed_invalidates = (
|
|
4527
|
+
"## Invalidates\n\n"
|
|
4528
|
+
'- I1: "the wording that is now wrong" — README.md. Updated by: 1.1\n'
|
|
4529
|
+
)
|
|
4530
|
+
start_task = section_boundary_task("1.1", checked=False)
|
|
4531
|
+
started = {
|
|
4532
|
+
position: start(
|
|
4533
|
+
f"invalidates-closed-{position}",
|
|
4534
|
+
section_boundary_tasks_md(
|
|
4535
|
+
section=closed_invalidates, task=start_task, position=position
|
|
4536
|
+
),
|
|
4537
|
+
)
|
|
4538
|
+
for position in ("above", "tail")
|
|
4539
|
+
}
|
|
4540
|
+
for position, payload in started.items():
|
|
4541
|
+
if not readable(payload, f"invalidates-closed-{position}", "task-start"):
|
|
4542
|
+
return 1
|
|
4543
|
+
if payload.get("status") != "pass":
|
|
4544
|
+
report(
|
|
4545
|
+
f"{label} refused a closed invalidation declaration with "
|
|
4546
|
+
f"the section {position} the task list."
|
|
4547
|
+
)
|
|
4548
|
+
report(repr(payload.get("problems")))
|
|
4549
|
+
return 1
|
|
4550
|
+
|
|
4551
|
+
# Cell 4: the silent half of the other reader. This is the cell that
|
|
4552
|
+
# returned `pass` before the boundary gained its task half.
|
|
4553
|
+
open_invalidates = (
|
|
4554
|
+
"## Invalidates\n\n"
|
|
4555
|
+
'- I1: "the wording that is now wrong" — README.md.\n'
|
|
4556
|
+
)
|
|
4557
|
+
open_start_task = section_boundary_task(
|
|
4558
|
+
"1.1", checked=False, repo_action=True
|
|
4559
|
+
)
|
|
4560
|
+
unstarted = {
|
|
4561
|
+
position: start(
|
|
4562
|
+
f"invalidates-open-{position}",
|
|
4563
|
+
section_boundary_tasks_md(
|
|
4564
|
+
section=open_invalidates, task=open_start_task, position=position
|
|
4565
|
+
),
|
|
4566
|
+
)
|
|
4567
|
+
for position in ("above", "tail")
|
|
4568
|
+
}
|
|
4569
|
+
for position, payload in unstarted.items():
|
|
4570
|
+
if not readable(payload, f"invalidates-open-{position}", "task-start"):
|
|
4571
|
+
return 1
|
|
4572
|
+
named = [
|
|
4573
|
+
message
|
|
4574
|
+
for message in closure_problems(payload, "invalidation-closure")
|
|
4575
|
+
if "I1" in message
|
|
4576
|
+
]
|
|
4577
|
+
if not named:
|
|
4578
|
+
report(
|
|
4579
|
+
f"{label} accepted an invalidation that closes nothing, "
|
|
4580
|
+
f"with the section {position} the task list. A task field "
|
|
4581
|
+
"of `none` is not the section's `- None.`."
|
|
4582
|
+
)
|
|
4583
|
+
report(repr(payload.get("problems")))
|
|
4584
|
+
return 1
|
|
4585
|
+
if problems_of(unstarted["above"]) != problems_of(unstarted["tail"]):
|
|
4586
|
+
report(
|
|
4587
|
+
f"{label} returned different problems for an identical unclosed "
|
|
4588
|
+
"invalidation depending on where the section sits."
|
|
4589
|
+
)
|
|
4590
|
+
report(repr(problems_of(unstarted["above"])))
|
|
4591
|
+
report(repr(problems_of(unstarted["tail"])))
|
|
4592
|
+
return 1
|
|
4593
|
+
|
|
4594
|
+
report(f"{label} scenario passed.")
|
|
4595
|
+
return 0
|
|
4596
|
+
|
|
4597
|
+
|
|
4598
|
+
SCHEMA_COPY_PAIRS = (
|
|
4599
|
+
(
|
|
4600
|
+
"openspec/schemas/keel-spec-driven/templates/tasks.md",
|
|
4601
|
+
"assets/openspec/schemas/keel-spec-driven/templates/tasks.md",
|
|
4602
|
+
),
|
|
4603
|
+
(
|
|
4604
|
+
"openspec/schemas/keel-spec-driven/schema.yaml",
|
|
4605
|
+
"assets/openspec/schemas/keel-spec-driven/schema.yaml",
|
|
4606
|
+
),
|
|
4607
|
+
)
|
|
4608
|
+
|
|
4609
|
+
|
|
4610
|
+
def validate_dry_run_overlay_accounting_scenario() -> int:
|
|
4611
|
+
label = "dry-run-overlay-accounting"
|
|
4612
|
+
|
|
4613
|
+
# A dry run is relied on, so it is wrong in both directions: naming a write
|
|
4614
|
+
# that will not happen trains the reader to ignore it, and omitting one
|
|
4615
|
+
# breaks the promise the dry run exists to make. `--check` used to omit the
|
|
4616
|
+
# overlay step entirely while `--install --dry-run` claimed every surface.
|
|
4617
|
+
def overlay_lines(text: str) -> list[str]:
|
|
4618
|
+
return [line for line in text.splitlines() if "overlay" in line]
|
|
4619
|
+
|
|
4620
|
+
def counts(text: str) -> str | None:
|
|
4621
|
+
found = re.search(r"refreshed=(\d+) current=(\d+) missing=(\d+)", text)
|
|
4622
|
+
return found.group(0) if found else None
|
|
4623
|
+
|
|
4624
|
+
with tempfile.TemporaryDirectory(prefix="keel-dry-run-overlay-") as raw_tmp:
|
|
4625
|
+
repo = Path(raw_tmp) / "repo"
|
|
4626
|
+
repo.mkdir()
|
|
4627
|
+
# The overlay surfaces are files OpenSpec generates; install merges into
|
|
4628
|
+
# them and skips the ones that are absent. Create them so this scenario
|
|
4629
|
+
# exercises the classification rather than the missing branch.
|
|
4630
|
+
for relative in (
|
|
4631
|
+
".claude/skills/openspec-propose/SKILL.md",
|
|
4632
|
+
".claude/skills/openspec-apply-change/SKILL.md",
|
|
4633
|
+
".claude/skills/openspec-archive-change/SKILL.md",
|
|
4634
|
+
".claude/commands/opsx/propose.md",
|
|
4635
|
+
".claude/commands/opsx/apply.md",
|
|
4636
|
+
".claude/commands/opsx/archive.md",
|
|
4637
|
+
):
|
|
4638
|
+
write_text(repo / relative, "# OpenSpec surface\n\nGenerated body.\n")
|
|
4639
|
+
if run_keel(repo, "--install", "--target", "claude").returncode != 0:
|
|
4640
|
+
report(f"{label} could not install a fixture repository.")
|
|
4641
|
+
return 1
|
|
4642
|
+
|
|
4643
|
+
# Nothing stale: neither dry run may name a file.
|
|
4644
|
+
check = run_keel(repo, "--check", "--target", "claude")
|
|
4645
|
+
if any("would refresh OpenSpec" in line and ".md" in line
|
|
4646
|
+
for line in overlay_lines(check.stdout)):
|
|
3889
4647
|
report(f"{label} named a surface that would not change.")
|
|
3890
4648
|
report("\n".join(overlay_lines(check.stdout)))
|
|
3891
4649
|
return 1
|
|
@@ -4424,84 +5182,357 @@ def validate_regression_check_tag_scenario() -> int:
|
|
|
4424
5182
|
return 0
|
|
4425
5183
|
|
|
4426
5184
|
|
|
4427
|
-
def
|
|
4428
|
-
|
|
4429
|
-
|
|
4430
|
-
|
|
4431
|
-
|
|
4432
|
-
|
|
4433
|
-
|
|
4434
|
-
|
|
4435
|
-
|
|
4436
|
-
|
|
4437
|
-
|
|
4438
|
-
|
|
4439
|
-
|
|
4440
|
-
|
|
4441
|
-
|
|
4442
|
-
|
|
4443
|
-
|
|
4444
|
-
|
|
4445
|
-
|
|
4446
|
-
|
|
5185
|
+
def change_verify_task(
|
|
5186
|
+
task_id: str,
|
|
5187
|
+
*,
|
|
5188
|
+
checked: bool,
|
|
5189
|
+
commands: tuple[str, ...],
|
|
5190
|
+
evidence: tuple[str, ...],
|
|
5191
|
+
) -> str:
|
|
5192
|
+
command_lines = "".join(f" - {item}\n" for item in commands)
|
|
5193
|
+
evidence_lines = "".join(f" - {item}\n" for item in evidence)
|
|
5194
|
+
return (
|
|
5195
|
+
f"- [{'x' if checked else ' '}] {task_id} Change verify fixture\n"
|
|
5196
|
+
" - Owner: claude\n"
|
|
5197
|
+
" - Mode: implementation\n"
|
|
5198
|
+
" - Covers:\n"
|
|
5199
|
+
" - E1: the task proves its own behavior\n"
|
|
5200
|
+
" - Read:\n"
|
|
5201
|
+
" - README.md\n"
|
|
5202
|
+
" - Touch:\n"
|
|
5203
|
+
" - src/example.js\n"
|
|
5204
|
+
" - Verify:\n"
|
|
5205
|
+
" - Strategy: vertical-tdd\n"
|
|
5206
|
+
f"{command_lines}"
|
|
5207
|
+
" - Acceptance:\n"
|
|
5208
|
+
" - Public behavior passes.\n"
|
|
5209
|
+
" - Autonomy boundary:\n"
|
|
5210
|
+
" - Default: hard-stop\n"
|
|
5211
|
+
" - Pre-authorized fallback: none\n"
|
|
5212
|
+
" - Stop Rules:\n"
|
|
5213
|
+
" - Stop on failure.\n"
|
|
5214
|
+
" - Evidence:\n"
|
|
5215
|
+
" - Contract: pending\n"
|
|
5216
|
+
f"{evidence_lines}"
|
|
5217
|
+
" - Review:\n"
|
|
5218
|
+
" - Status: pass\n"
|
|
5219
|
+
" - Acceptance check: behavior proven.\n"
|
|
5220
|
+
" - Scope check: writes stayed inside Touch.\n"
|
|
5221
|
+
" - Findings: none\n"
|
|
5222
|
+
" - Blocker: none\n"
|
|
5223
|
+
" - Stop if:\n"
|
|
5224
|
+
" - Requires files outside Touch.\n"
|
|
5225
|
+
)
|
|
4447
5226
|
|
|
4448
|
-
derived = packaged_openspec_schema_install_paths()
|
|
4449
|
-
if not derived:
|
|
4450
|
-
report(
|
|
4451
|
-
f"{label} derived no packaged schema paths, so every assertion that "
|
|
4452
|
-
"iterates them verifies nothing."
|
|
4453
|
-
)
|
|
4454
|
-
return 1
|
|
4455
5227
|
|
|
4456
|
-
|
|
4457
|
-
|
|
4458
|
-
|
|
4459
|
-
|
|
4460
|
-
|
|
4461
|
-
|
|
4462
|
-
report((install.stderr or install.stdout).strip())
|
|
4463
|
-
return 1
|
|
5228
|
+
def change_verify_deferred_evidence_repo(root: Path, name: str) -> Path:
|
|
5229
|
+
repo = root / name
|
|
5230
|
+
write_text(repo / "openspec/changes/demo/proposal.md", "# Proposal\n")
|
|
5231
|
+
write_text(repo / "openspec/changes/demo/design.md", "## Context\n\nfixture\n")
|
|
5232
|
+
write_text(repo / "openspec/changes/demo/specs/demo/spec.md", "## ADDED Requirements\n")
|
|
5233
|
+
return repo
|
|
4464
5234
|
|
|
4465
|
-
schema_root = repo / OPENSPEC_SCHEMA_ROOT
|
|
4466
|
-
installed = sorted(
|
|
4467
|
-
path.relative_to(repo).as_posix()
|
|
4468
|
-
for path in schema_root.rglob("*")
|
|
4469
|
-
if path.is_file()
|
|
4470
|
-
)
|
|
4471
|
-
if installed != sorted(derived):
|
|
4472
|
-
report(
|
|
4473
|
-
f"{label} derived paths do not match what keel --install wrote."
|
|
4474
|
-
)
|
|
4475
|
-
report(f"derived: {sorted(derived)}")
|
|
4476
|
-
report(f"installed: {installed}")
|
|
4477
|
-
return 1
|
|
4478
5235
|
|
|
4479
|
-
|
|
4480
|
-
|
|
5236
|
+
def validate_change_verify_deferred_evidence_scenario() -> int:
|
|
5237
|
+
"""A `(regression)`-tagged check may defer its bare Evidence to a
|
|
5238
|
+
change-level `C<n>` check, run once for the whole change instead of once
|
|
5239
|
+
per task. Issue #95: a full regression suite that only needs to run once
|
|
5240
|
+
had no place to live in `tasks.md`, so authors either paid to repeat it on
|
|
5241
|
+
every task or gave the middle tasks no safety net at all.
|
|
5242
|
+
"""
|
|
5243
|
+
label = "change-verify-deferred-evidence"
|
|
4481
5244
|
|
|
5245
|
+
change_verify_c1 = (
|
|
5246
|
+
"## Change Verify\n\n"
|
|
5247
|
+
"- Strategy: regression-first\n"
|
|
5248
|
+
"- C1: the full three-layer suite is 0 failed\n\n"
|
|
5249
|
+
)
|
|
5250
|
+
change_evidence_c1_pending = "## Change Evidence\n\n- C1: pending\n\n"
|
|
5251
|
+
change_evidence_c1_pass = (
|
|
5252
|
+
"## Change Evidence\n\n- C1: pass. Full suite ran clean at 142/142.\n\n"
|
|
5253
|
+
)
|
|
5254
|
+
tagged_commands = (
|
|
5255
|
+
"M1: behavior reaches the public interface",
|
|
5256
|
+
"M2 (regression): the full suite stays green",
|
|
5257
|
+
)
|
|
5258
|
+
deferred_evidence = (
|
|
5259
|
+
"M1: behavior exercised.",
|
|
5260
|
+
"M1.red: failed before the implementation.",
|
|
5261
|
+
"M1.green: passed after.",
|
|
5262
|
+
"M2: deferred to C1",
|
|
5263
|
+
)
|
|
4482
5264
|
|
|
4483
|
-
def
|
|
4484
|
-
|
|
5265
|
+
def tasks_md(
|
|
5266
|
+
*,
|
|
5267
|
+
change_verify: str = "",
|
|
5268
|
+
change_evidence: str = "",
|
|
5269
|
+
commands: tuple[str, ...] = tagged_commands,
|
|
5270
|
+
evidence: tuple[str, ...],
|
|
5271
|
+
checked: bool = True,
|
|
5272
|
+
) -> str:
|
|
5273
|
+
sections = (
|
|
5274
|
+
"## Invalidates\n\n- None.\n\n"
|
|
5275
|
+
"## Expectation Coverage\n\n"
|
|
5276
|
+
"- E1: the task proves its own behavior. Covered by: 1.1\n\n"
|
|
5277
|
+
+ change_verify
|
|
5278
|
+
+ change_evidence
|
|
5279
|
+
)
|
|
5280
|
+
return "# Tasks\n\n" + sections + change_verify_task(
|
|
5281
|
+
"1.1", checked=checked, commands=commands, evidence=evidence
|
|
5282
|
+
)
|
|
4485
5283
|
|
|
4486
|
-
|
|
4487
|
-
|
|
4488
|
-
|
|
4489
|
-
|
|
4490
|
-
|
|
4491
|
-
|
|
4492
|
-
local_text = (ROOT / local).read_text(encoding="utf-8")
|
|
4493
|
-
packaged_text = (ROOT / packaged).read_text(encoding="utf-8")
|
|
4494
|
-
if local_text != packaged_text:
|
|
4495
|
-
report(f"{label} schema copies diverge: {local} vs {packaged}")
|
|
4496
|
-
return 1
|
|
5284
|
+
with tempfile.TemporaryDirectory(
|
|
5285
|
+
prefix="keel-change-verify-", ignore_cleanup_errors=True
|
|
5286
|
+
) as raw:
|
|
5287
|
+
tmp = Path(raw)
|
|
5288
|
+
repo = change_verify_deferred_evidence_repo(tmp, "repo")
|
|
5289
|
+
tasks_path = repo / "openspec/changes/demo/tasks.md"
|
|
4497
5290
|
|
|
4498
|
-
|
|
4499
|
-
|
|
4500
|
-
|
|
4501
|
-
|
|
4502
|
-
|
|
4503
|
-
|
|
4504
|
-
|
|
5291
|
+
def codes(payload: dict) -> set[str]:
|
|
5292
|
+
return {item.get("code") for item in payload.get("problems", [])}
|
|
5293
|
+
|
|
5294
|
+
def complete(fixture: str) -> dict:
|
|
5295
|
+
write_text(tasks_path, fixture)
|
|
5296
|
+
record_contract_anchor(repo, "demo")
|
|
5297
|
+
result = run_keel(
|
|
5298
|
+
repo, "gate", "task-complete", "--change", "demo", "--task", "1.1", "--json"
|
|
5299
|
+
)
|
|
5300
|
+
return json.loads(result.stdout)
|
|
5301
|
+
|
|
5302
|
+
def close(fixture: str) -> dict:
|
|
5303
|
+
write_text(tasks_path, fixture)
|
|
5304
|
+
record_contract_anchor(repo, "demo")
|
|
5305
|
+
result = run_keel(
|
|
5306
|
+
repo, "gate", "change-close", "--change", "demo", "--action", "sync", "--json"
|
|
5307
|
+
)
|
|
5308
|
+
return json.loads(result.stdout)
|
|
5309
|
+
|
|
5310
|
+
# (a) A `(regression)`-tagged check deferring to a declared C1 passes
|
|
5311
|
+
# task-complete with no C1 result required yet — the change has not
|
|
5312
|
+
# closed, so the deferred check may legitimately not have run.
|
|
5313
|
+
payload = complete(tasks_md(change_verify=change_verify_c1, evidence=deferred_evidence))
|
|
5314
|
+
if payload.get("status") != "pass":
|
|
5315
|
+
report(f"{label} refused a regression check deferring to a declared change-level check.")
|
|
5316
|
+
report(json.dumps(payload.get("problems", []), indent=2))
|
|
5317
|
+
return 1
|
|
5318
|
+
|
|
5319
|
+
# (b) An untagged check deferring to C1 fails: only a
|
|
5320
|
+
# `(regression)`-tagged check may defer.
|
|
5321
|
+
payload = complete(
|
|
5322
|
+
tasks_md(
|
|
5323
|
+
change_verify=change_verify_c1,
|
|
5324
|
+
commands=(
|
|
5325
|
+
"M1: behavior reaches the public interface",
|
|
5326
|
+
"M2: the full suite stays green",
|
|
5327
|
+
),
|
|
5328
|
+
evidence=(
|
|
5329
|
+
"M1: behavior exercised.",
|
|
5330
|
+
"M1.red: failed before the implementation.",
|
|
5331
|
+
"M1.green: passed after.",
|
|
5332
|
+
"M2: deferred to C1",
|
|
5333
|
+
"M2.red: n/a",
|
|
5334
|
+
"M2.green: n/a",
|
|
5335
|
+
),
|
|
5336
|
+
)
|
|
5337
|
+
)
|
|
5338
|
+
if (
|
|
5339
|
+
payload.get("status") != "fail"
|
|
5340
|
+
or "deferred-evidence-not-regression" not in codes(payload)
|
|
5341
|
+
):
|
|
5342
|
+
report(f"{label} let an untagged check defer to a change-level check.")
|
|
5343
|
+
report(json.dumps(payload, indent=2))
|
|
5344
|
+
return 1
|
|
5345
|
+
|
|
5346
|
+
# (c) A `(regression)`-tagged check deferring to an undeclared C9
|
|
5347
|
+
# fails: the reference must resolve.
|
|
5348
|
+
payload = complete(
|
|
5349
|
+
tasks_md(
|
|
5350
|
+
change_verify=change_verify_c1,
|
|
5351
|
+
evidence=(
|
|
5352
|
+
"M1: behavior exercised.",
|
|
5353
|
+
"M1.red: failed before the implementation.",
|
|
5354
|
+
"M1.green: passed after.",
|
|
5355
|
+
"M2: deferred to C9",
|
|
5356
|
+
),
|
|
5357
|
+
)
|
|
5358
|
+
)
|
|
5359
|
+
if payload.get("status") != "fail" or "deferred-check-unresolved" not in codes(payload):
|
|
5360
|
+
report(f"{label} accepted a deferral to a change-level check that was never declared.")
|
|
5361
|
+
report(json.dumps(payload, indent=2))
|
|
5362
|
+
return 1
|
|
5363
|
+
|
|
5364
|
+
# (d) change-close fails the same way when a task defers but
|
|
5365
|
+
# `## Change Verify` does not exist at all.
|
|
5366
|
+
payload = close(tasks_md(evidence=deferred_evidence))
|
|
5367
|
+
if payload.get("status") != "fail" or "deferred-check-unresolved" not in codes(payload):
|
|
5368
|
+
report(f"{label} closed a change whose deferred check has no Change Verify section.")
|
|
5369
|
+
report(json.dumps(payload, indent=2))
|
|
5370
|
+
return 1
|
|
5371
|
+
|
|
5372
|
+
# (e) change-close fails when C1 is declared but Change Evidence has
|
|
5373
|
+
# no concrete entry for it — the check has to actually run once.
|
|
5374
|
+
payload = close(
|
|
5375
|
+
tasks_md(
|
|
5376
|
+
change_verify=change_verify_c1,
|
|
5377
|
+
change_evidence=change_evidence_c1_pending,
|
|
5378
|
+
evidence=deferred_evidence,
|
|
5379
|
+
)
|
|
5380
|
+
)
|
|
5381
|
+
if payload.get("status") != "fail" or "change-evidence-missing" not in codes(payload):
|
|
5382
|
+
report(f"{label} closed a change whose declared C1 has no concrete Change Evidence.")
|
|
5383
|
+
report(json.dumps(payload, indent=2))
|
|
5384
|
+
return 1
|
|
5385
|
+
|
|
5386
|
+
# (f) change-close passes once Change Evidence carries concrete
|
|
5387
|
+
# evidence for every declared C<n>.
|
|
5388
|
+
payload = close(
|
|
5389
|
+
tasks_md(
|
|
5390
|
+
change_verify=change_verify_c1,
|
|
5391
|
+
change_evidence=change_evidence_c1_pass,
|
|
5392
|
+
evidence=deferred_evidence,
|
|
5393
|
+
)
|
|
5394
|
+
)
|
|
5395
|
+
if payload.get("status") != "pass":
|
|
5396
|
+
report(f"{label} refused a fully resolved deferred change-level check at close.")
|
|
5397
|
+
report(json.dumps(payload.get("problems", []), indent=2))
|
|
5398
|
+
return 1
|
|
5399
|
+
|
|
5400
|
+
# (g) A change no task defers in reports nothing about either
|
|
5401
|
+
# section, whether present or absent — the feature costs nothing to a
|
|
5402
|
+
# change that never uses it, and a declared-but-unused section is
|
|
5403
|
+
# still held to its own shape.
|
|
5404
|
+
undeferred_evidence = (
|
|
5405
|
+
"M1: behavior exercised.",
|
|
5406
|
+
"M1.red: failed before the implementation.",
|
|
5407
|
+
"M1.green: passed after.",
|
|
5408
|
+
"M2: existing suite still green.",
|
|
5409
|
+
)
|
|
5410
|
+
without_sections = close(tasks_md(evidence=undeferred_evidence))
|
|
5411
|
+
with_unused_sections = close(
|
|
5412
|
+
tasks_md(
|
|
5413
|
+
change_verify=change_verify_c1,
|
|
5414
|
+
change_evidence=change_evidence_c1_pass,
|
|
5415
|
+
evidence=undeferred_evidence,
|
|
5416
|
+
)
|
|
5417
|
+
)
|
|
5418
|
+
for payload in (without_sections, with_unused_sections):
|
|
5419
|
+
stray = [
|
|
5420
|
+
item
|
|
5421
|
+
for item in payload.get("problems", [])
|
|
5422
|
+
if item.get("code", "").startswith("change-verify")
|
|
5423
|
+
or item.get("code", "").startswith("change-evidence")
|
|
5424
|
+
or item.get("code", "").startswith("deferred-")
|
|
5425
|
+
]
|
|
5426
|
+
if stray:
|
|
5427
|
+
report(
|
|
5428
|
+
f"{label} reported a Change Verify/Evidence problem for a "
|
|
5429
|
+
"change nothing defers to."
|
|
5430
|
+
)
|
|
5431
|
+
report(json.dumps(stray, indent=2))
|
|
5432
|
+
return 1
|
|
5433
|
+
|
|
5434
|
+
# (h) Non-contiguous C<n> labels fail change-close: the section is
|
|
5435
|
+
# held to the same label discipline `M<n>` already is.
|
|
5436
|
+
broken_change_verify = (
|
|
5437
|
+
"## Change Verify\n\n"
|
|
5438
|
+
"- Strategy: regression-first\n"
|
|
5439
|
+
"- C1: the full three-layer suite is 0 failed\n"
|
|
5440
|
+
"- C3: the byte baseline stays identical\n\n"
|
|
5441
|
+
)
|
|
5442
|
+
payload = close(
|
|
5443
|
+
tasks_md(
|
|
5444
|
+
change_verify=broken_change_verify,
|
|
5445
|
+
change_evidence="## Change Evidence\n\n- C1: pass.\n- C3: pass.\n\n",
|
|
5446
|
+
evidence=deferred_evidence,
|
|
5447
|
+
)
|
|
5448
|
+
)
|
|
5449
|
+
if payload.get("status") != "fail" or "change-verify-shape" not in codes(payload):
|
|
5450
|
+
report(f"{label} accepted non-contiguous Change Verify labels.")
|
|
5451
|
+
report(json.dumps(payload, indent=2))
|
|
5452
|
+
return 1
|
|
5453
|
+
|
|
5454
|
+
report(f"{label} scenario passed.")
|
|
5455
|
+
return 0
|
|
5456
|
+
|
|
5457
|
+
|
|
5458
|
+
def validate_packaged_schema_derivation_scenario() -> int:
|
|
5459
|
+
label = "packaged-schema-derivation"
|
|
5460
|
+
|
|
5461
|
+
# The helper derives the consumer-repo paths every install/uninstall/clear
|
|
5462
|
+
# assertion iterates. When its root stopped existing it returned an empty
|
|
5463
|
+
# list, so those loops compared nothing and reported success. Anchor it to
|
|
5464
|
+
# what the installer really writes, and make emptiness a failure here.
|
|
5465
|
+
try:
|
|
5466
|
+
packaged_openspec_schema_install_paths(ROOT / "no-such-packaged-root")
|
|
5467
|
+
except FileNotFoundError as error:
|
|
5468
|
+
if "no-such-packaged-root" not in str(error):
|
|
5469
|
+
report(f"{label} missing-root failure does not name the path it expected.")
|
|
5470
|
+
report(str(error))
|
|
5471
|
+
return 1
|
|
5472
|
+
else:
|
|
5473
|
+
report(
|
|
5474
|
+
f"{label} returned a set for a missing packaged root instead of failing; "
|
|
5475
|
+
"an absent root must not silently empty its callers' assertions."
|
|
5476
|
+
)
|
|
5477
|
+
return 1
|
|
5478
|
+
|
|
5479
|
+
derived = packaged_openspec_schema_install_paths()
|
|
5480
|
+
if not derived:
|
|
5481
|
+
report(
|
|
5482
|
+
f"{label} derived no packaged schema paths, so every assertion that "
|
|
5483
|
+
"iterates them verifies nothing."
|
|
5484
|
+
)
|
|
5485
|
+
return 1
|
|
5486
|
+
|
|
5487
|
+
with tempfile.TemporaryDirectory(prefix="keel-packaged-schema-") as raw_tmp:
|
|
5488
|
+
repo = Path(raw_tmp) / "repo"
|
|
5489
|
+
repo.mkdir()
|
|
5490
|
+
install = run_keel(repo, "--install")
|
|
5491
|
+
if install.returncode != 0:
|
|
5492
|
+
report(f"{label} keel --install failed.")
|
|
5493
|
+
report((install.stderr or install.stdout).strip())
|
|
5494
|
+
return 1
|
|
5495
|
+
|
|
5496
|
+
schema_root = repo / OPENSPEC_SCHEMA_ROOT
|
|
5497
|
+
installed = sorted(
|
|
5498
|
+
path.relative_to(repo).as_posix()
|
|
5499
|
+
for path in schema_root.rglob("*")
|
|
5500
|
+
if path.is_file()
|
|
5501
|
+
)
|
|
5502
|
+
if installed != sorted(derived):
|
|
5503
|
+
report(
|
|
5504
|
+
f"{label} derived paths do not match what keel --install wrote."
|
|
5505
|
+
)
|
|
5506
|
+
report(f"derived: {sorted(derived)}")
|
|
5507
|
+
report(f"installed: {installed}")
|
|
5508
|
+
return 1
|
|
5509
|
+
|
|
5510
|
+
report(f"{label} scenario passed.")
|
|
5511
|
+
return 0
|
|
5512
|
+
|
|
5513
|
+
|
|
5514
|
+
def validate_invalidation_authoring_surface_scenario() -> int:
|
|
5515
|
+
label = "invalidation-authoring-surface"
|
|
5516
|
+
|
|
5517
|
+
# The two schema copies are the repo-local one OpenSpec resolves and the
|
|
5518
|
+
# packaged one `keel --init` writes. This is the only check that asserts they
|
|
5519
|
+
# agree: compact-task-authoring used to imply it through a projection loop
|
|
5520
|
+
# rooted at trees that no longer exist, so it compared nothing and has since
|
|
5521
|
+
# been removed.
|
|
5522
|
+
for local, packaged in SCHEMA_COPY_PAIRS:
|
|
5523
|
+
local_text = (ROOT / local).read_text(encoding="utf-8")
|
|
5524
|
+
packaged_text = (ROOT / packaged).read_text(encoding="utf-8")
|
|
5525
|
+
if local_text != packaged_text:
|
|
5526
|
+
report(f"{label} schema copies diverge: {local} vs {packaged}")
|
|
5527
|
+
return 1
|
|
5528
|
+
|
|
5529
|
+
template = (
|
|
5530
|
+
ROOT / "openspec/schemas/keel-spec-driven/templates/tasks.md"
|
|
5531
|
+
).read_text(encoding="utf-8")
|
|
5532
|
+
for marker in ("## Invalidates", "- None.", "- I1:"):
|
|
5533
|
+
if marker not in template:
|
|
5534
|
+
report(f"{label} tasks template lacks the invalidation section: {marker}")
|
|
5535
|
+
return 1
|
|
4505
5536
|
|
|
4506
5537
|
schema = (
|
|
4507
5538
|
ROOT / "openspec/schemas/keel-spec-driven/schema.yaml"
|
|
@@ -5124,7 +6155,10 @@ def validate_covers_separator_collision_scenario() -> int:
|
|
|
5124
6155
|
)
|
|
5125
6156
|
report(trimmed_messages)
|
|
5126
6157
|
return 1
|
|
5127
|
-
# A capability with no
|
|
6158
|
+
# A capability with no colliding name receives no separator hint. It
|
|
6159
|
+
# still gets the unresolved diagnostic, which since 5.21.0 names what
|
|
6160
|
+
# the spec holds instead; what is asserted here is only that the
|
|
6161
|
+
# separator explanation stays with the capability that has one.
|
|
5128
6162
|
plain = start(
|
|
5129
6163
|
"plain", "clean-cap / No such requirement / No such scenario"
|
|
5130
6164
|
)
|
|
@@ -5162,6 +6196,195 @@ def validate_covers_separator_collision_scenario() -> int:
|
|
|
5162
6196
|
return 0
|
|
5163
6197
|
|
|
5164
6198
|
|
|
6199
|
+
def validate_unresolved_covers_names_what_failed_scenario() -> int:
|
|
6200
|
+
"""Issue #49, 2026-08-02 supplement: an unresolved reference said only that.
|
|
6201
|
+
|
|
6202
|
+
A two-segment `Covers` reference whose second segment is a Scenario name is
|
|
6203
|
+
the most common way to write the notation wrong — the shipped task template
|
|
6204
|
+
taught it — and the spec being read holds that exact name one heading level
|
|
6205
|
+
down. The refusal reported the reference back and nothing else, while the
|
|
6206
|
+
hierarchy sentence sat thirty lines above in the same function, reachable
|
|
6207
|
+
only by over-segmenting. Every case below still fails; only what it says
|
|
6208
|
+
changes, and the last case asserts that.
|
|
6209
|
+
"""
|
|
6210
|
+
spec = (
|
|
6211
|
+
"# demo-cap\n\n## Purpose\nDemo.\n\n"
|
|
6212
|
+
"### Requirement: The store validates itself\n"
|
|
6213
|
+
"Keel MUST validate the published store.\n\n"
|
|
6214
|
+
"#### Scenario: A published store passes the pinned validator\n"
|
|
6215
|
+
"- **WHEN** the pinned validator runs\n"
|
|
6216
|
+
"- **THEN** the published store passes\n"
|
|
6217
|
+
"- **AND THEN** the pin is reported\n\n"
|
|
6218
|
+
"#### Scenario: A shared scenario name\n"
|
|
6219
|
+
"- **WHEN** a thing\n- **THEN** another\n\n"
|
|
6220
|
+
"### Requirement: Another requirement\n"
|
|
6221
|
+
"Keel MUST do the other thing.\n\n"
|
|
6222
|
+
"#### Scenario: A shared scenario name\n"
|
|
6223
|
+
"- **WHEN** a thing\n- **THEN** another\n"
|
|
6224
|
+
)
|
|
6225
|
+
|
|
6226
|
+
with tempfile.TemporaryDirectory(prefix="keel-covers-what-failed-") as raw:
|
|
6227
|
+
repo = Path(raw)
|
|
6228
|
+
write_text(repo / "openspec/specs/demo-cap/spec.md", spec)
|
|
6229
|
+
|
|
6230
|
+
def start(change: str, reference: str) -> dict:
|
|
6231
|
+
write_text(
|
|
6232
|
+
repo / f"openspec/changes/{change}/tasks.md",
|
|
6233
|
+
task_capsule_compact_fixture().replace(
|
|
6234
|
+
" - E1: Public behavior passes.\n", f" - {reference}\n"
|
|
6235
|
+
),
|
|
6236
|
+
)
|
|
6237
|
+
result = run_keel(
|
|
6238
|
+
repo, "gate", "task-start", "--change", change, "--task", "1.1",
|
|
6239
|
+
"--json",
|
|
6240
|
+
)
|
|
6241
|
+
return json.loads(result.stdout)
|
|
6242
|
+
|
|
6243
|
+
def refusal(change: str, reference: str) -> tuple[dict, str]:
|
|
6244
|
+
payload = start(change, reference)
|
|
6245
|
+
message = " ".join(
|
|
6246
|
+
problem.get("message", "")
|
|
6247
|
+
for problem in payload.get("problems", [])
|
|
6248
|
+
)
|
|
6249
|
+
return payload, message
|
|
6250
|
+
|
|
6251
|
+
# The reported case: the Scenario name written where a Requirement goes.
|
|
6252
|
+
scenario_ref = "demo-cap / A published store passes the pinned validator"
|
|
6253
|
+
offered, offered_message = refusal("offered", scenario_ref)
|
|
6254
|
+
corrected = (
|
|
6255
|
+
"demo-cap / The store validates itself"
|
|
6256
|
+
" / A published store passes the pinned validator"
|
|
6257
|
+
)
|
|
6258
|
+
if offered.get("status") != "fail":
|
|
6259
|
+
report(
|
|
6260
|
+
"unresolved-covers-names-what-failed: a Scenario named as a "
|
|
6261
|
+
"Requirement stopped being refused."
|
|
6262
|
+
)
|
|
6263
|
+
report(json.dumps(offered.get("problems"), ensure_ascii=False))
|
|
6264
|
+
return 1
|
|
6265
|
+
if not any(
|
|
6266
|
+
problem.get("code") == "unresolved-covers"
|
|
6267
|
+
for problem in offered.get("problems", [])
|
|
6268
|
+
):
|
|
6269
|
+
report(
|
|
6270
|
+
"unresolved-covers-names-what-failed: the refusal no longer "
|
|
6271
|
+
"carries the unresolved-covers code."
|
|
6272
|
+
)
|
|
6273
|
+
report(json.dumps(offered.get("problems"), ensure_ascii=False))
|
|
6274
|
+
return 1
|
|
6275
|
+
if (
|
|
6276
|
+
"The store validates itself" not in offered_message
|
|
6277
|
+
or "Scenario" not in offered_message
|
|
6278
|
+
or corrected not in offered_message
|
|
6279
|
+
):
|
|
6280
|
+
report(
|
|
6281
|
+
"unresolved-covers-names-what-failed: the diagnostic did not "
|
|
6282
|
+
"name the Requirement the Scenario belongs to, did not call it "
|
|
6283
|
+
"a Scenario, or did not spell the corrected reference."
|
|
6284
|
+
)
|
|
6285
|
+
report(offered_message)
|
|
6286
|
+
return 1
|
|
6287
|
+
|
|
6288
|
+
# A name the capability declares nowhere: say the spec was read.
|
|
6289
|
+
absent, absent_message = refusal("absent", "demo-cap / No such name")
|
|
6290
|
+
if absent.get("status") != "fail":
|
|
6291
|
+
report(
|
|
6292
|
+
"unresolved-covers-names-what-failed: a name the capability does "
|
|
6293
|
+
"not declare stopped being refused."
|
|
6294
|
+
)
|
|
6295
|
+
report(json.dumps(absent.get("problems"), ensure_ascii=False))
|
|
6296
|
+
return 1
|
|
6297
|
+
if (
|
|
6298
|
+
"No such name" not in absent_message
|
|
6299
|
+
or "hierarchy is capability / requirement" not in absent_message
|
|
6300
|
+
):
|
|
6301
|
+
report(
|
|
6302
|
+
"unresolved-covers-names-what-failed: a name the spec does not "
|
|
6303
|
+
"declare was not reported as read, or the hierarchy was withheld."
|
|
6304
|
+
)
|
|
6305
|
+
report(absent_message)
|
|
6306
|
+
return 1
|
|
6307
|
+
|
|
6308
|
+
# A capability with no spec at all is a different failure.
|
|
6309
|
+
nospec, nospec_message = refusal("nospec", "nosuch-cap / Whatever")
|
|
6310
|
+
if nospec.get("status") != "fail":
|
|
6311
|
+
report(
|
|
6312
|
+
"unresolved-covers-names-what-failed: a capability with no spec "
|
|
6313
|
+
"stopped being refused."
|
|
6314
|
+
)
|
|
6315
|
+
report(json.dumps(nospec.get("problems"), ensure_ascii=False))
|
|
6316
|
+
return 1
|
|
6317
|
+
if (
|
|
6318
|
+
"nosuch-cap" not in nospec_message
|
|
6319
|
+
or "no spec" not in nospec_message.lower()
|
|
6320
|
+
):
|
|
6321
|
+
report(
|
|
6322
|
+
"unresolved-covers-names-what-failed: a capability with no spec "
|
|
6323
|
+
"was not distinguished from a name the spec lacks."
|
|
6324
|
+
)
|
|
6325
|
+
report(nospec_message)
|
|
6326
|
+
return 1
|
|
6327
|
+
|
|
6328
|
+
# Ambiguous: the same Scenario name under two Requirements has no single
|
|
6329
|
+
# correction, so none is offered.
|
|
6330
|
+
shared, shared_message = refusal("shared", "demo-cap / A shared scenario name")
|
|
6331
|
+
if shared.get("status") != "fail":
|
|
6332
|
+
report(
|
|
6333
|
+
"unresolved-covers-names-what-failed: an ambiguous Scenario name "
|
|
6334
|
+
"stopped being refused."
|
|
6335
|
+
)
|
|
6336
|
+
report(json.dumps(shared.get("problems"), ensure_ascii=False))
|
|
6337
|
+
return 1
|
|
6338
|
+
if (
|
|
6339
|
+
"The store validates itself" not in shared_message
|
|
6340
|
+
or "Another requirement" not in shared_message
|
|
6341
|
+
):
|
|
6342
|
+
report(
|
|
6343
|
+
"unresolved-covers-names-what-failed: an ambiguous Scenario name "
|
|
6344
|
+
"did not name the Requirements it appears under."
|
|
6345
|
+
)
|
|
6346
|
+
report(shared_message)
|
|
6347
|
+
return 1
|
|
6348
|
+
if "Write it as" in shared_message:
|
|
6349
|
+
report(
|
|
6350
|
+
"unresolved-covers-names-what-failed: a corrected reference was "
|
|
6351
|
+
"offered for a Scenario name that has more than one parent."
|
|
6352
|
+
)
|
|
6353
|
+
report(shared_message)
|
|
6354
|
+
return 1
|
|
6355
|
+
|
|
6356
|
+
# What resolves is unchanged. This is the assertion that must not be
|
|
6357
|
+
# dropped: reading more of the spec on the failure path is the direction
|
|
6358
|
+
# in which a refusal could accidentally become a resolution.
|
|
6359
|
+
resolves = start("resolves", "demo-cap / The store validates itself")
|
|
6360
|
+
capsule = resolves.get("contract", {}).get("capsule", {})
|
|
6361
|
+
authority = capsule.get("authority", [])
|
|
6362
|
+
if (
|
|
6363
|
+
resolves.get("status") != "pass"
|
|
6364
|
+
or resolves.get("problems")
|
|
6365
|
+
or len(authority) != 1
|
|
6366
|
+
or authority[0].get("kind") != "requirement"
|
|
6367
|
+
or not authority[0]
|
|
6368
|
+
.get("source", "")
|
|
6369
|
+
.endswith("specs/demo-cap/spec.md#Requirement:The store validates itself")
|
|
6370
|
+
):
|
|
6371
|
+
report(
|
|
6372
|
+
"unresolved-covers-names-what-failed: a reference that resolves "
|
|
6373
|
+
"no longer compiles to the same authority."
|
|
6374
|
+
)
|
|
6375
|
+
report(json.dumps(resolves, ensure_ascii=False)[:2000])
|
|
6376
|
+
return 1
|
|
6377
|
+
|
|
6378
|
+
if "unresolved-covers-names-what-failed" not in {name for name, _ in SCENARIOS}:
|
|
6379
|
+
report(
|
|
6380
|
+
"unresolved-covers-names-what-failed: the scenario registry does not "
|
|
6381
|
+
"include it."
|
|
6382
|
+
)
|
|
6383
|
+
return 1
|
|
6384
|
+
report("unresolved-covers-names-what-failed scenario passed.")
|
|
6385
|
+
return 0
|
|
6386
|
+
|
|
6387
|
+
|
|
5165
6388
|
def validate_unresolved_authority_names_field_scenario() -> int:
|
|
5166
6389
|
"""Issue #7 example 3: the diagnostic must name what it actually reads.
|
|
5167
6390
|
|
|
@@ -6216,6 +7439,14 @@ def validate_default_completion_attributes_writes_scenario() -> int:
|
|
|
6216
7439
|
dirty then answers "did this task write it" without asking Git to answer
|
|
6217
7440
|
"which task wrote it", which is the question it cannot answer in a
|
|
6218
7441
|
half-finished change.
|
|
7442
|
+
|
|
7443
|
+
Issue #72: recording only the path, not its content, meant a path already
|
|
7444
|
+
dirty at task start was exempt for the rest of the task's life, even after
|
|
7445
|
+
the task wrote it again — the common case, since most tasks in this
|
|
7446
|
+
repository start on a tree a prior task already legitimately touched. M2
|
|
7447
|
+
proves the fix: a dirty-at-start path the task modifies again is
|
|
7448
|
+
attributed, while M1's own dirty-at-start path (never touched again) stays
|
|
7449
|
+
exempt.
|
|
6219
7450
|
"""
|
|
6220
7451
|
label = "default-completion-attributes-writes"
|
|
6221
7452
|
|
|
@@ -6314,6 +7545,34 @@ def validate_default_completion_attributes_writes_scenario() -> int:
|
|
|
6314
7545
|
)
|
|
6315
7546
|
return 1
|
|
6316
7547
|
|
|
7548
|
+
# M2 — issue #72: a path already dirty at task start is exempt only
|
|
7549
|
+
# while its content stays the one recorded then. A task that goes on
|
|
7550
|
+
# to modify that same path has written it, and the boundary M1 proved
|
|
7551
|
+
# above must still catch that write instead of exempting it forever
|
|
7552
|
+
# because it happened to be dirty at the start too.
|
|
7553
|
+
write_text(
|
|
7554
|
+
repo / "src/already-dirty.js", "// touched again by the task\n"
|
|
7555
|
+
)
|
|
7556
|
+
payload = gate()
|
|
7557
|
+
problems = outside(payload)
|
|
7558
|
+
if not any("src/already-dirty.js" in message for message in problems):
|
|
7559
|
+
report(
|
|
7560
|
+
f"{label} M2 a path already dirty at task start, then "
|
|
7561
|
+
"modified again by the task, was not attributed. Recording "
|
|
7562
|
+
"only the path (not its content) at task start exempts every "
|
|
7563
|
+
"later write to it, not just the one that predates the task."
|
|
7564
|
+
)
|
|
7565
|
+
report(f" status={payload.get('status')!r}")
|
|
7566
|
+
for warning in payload.get("warnings", []):
|
|
7567
|
+
report(f" warning: {warning}")
|
|
7568
|
+
return 1
|
|
7569
|
+
if payload.get("status") != "fail":
|
|
7570
|
+
report(
|
|
7571
|
+
f"{label} M2 the gate named the re-touched path but did not "
|
|
7572
|
+
f"fail; got status {payload.get('status')!r}."
|
|
7573
|
+
)
|
|
7574
|
+
return 1
|
|
7575
|
+
|
|
6317
7576
|
# M1 — an explicit base answers the question the caller asked, which is
|
|
6318
7577
|
# the broader one: everything since that commit, including what was
|
|
6319
7578
|
# already dirty when the task started.
|
|
@@ -8604,106 +9863,623 @@ def validate_findings_resolved_here_scenario() -> int:
|
|
|
8604
9863
|
)
|
|
8605
9864
|
return 1
|
|
8606
9865
|
|
|
8607
|
-
as_owner = complete(
|
|
8608
|
-
f"the helper rewrote its own baseline path. Durable owner: {TRACKER_OWNER}"
|
|
9866
|
+
as_owner = complete(
|
|
9867
|
+
f"the helper rewrote its own baseline path. Durable owner: {TRACKER_OWNER}"
|
|
9868
|
+
)
|
|
9869
|
+
if as_owner.returncode != 0:
|
|
9870
|
+
report(
|
|
9871
|
+
"findings-resolved-here: M2 the same tracker reference must "
|
|
9872
|
+
"still pass as a durable owner."
|
|
9873
|
+
)
|
|
9874
|
+
report((as_owner.stderr or as_owner.stdout).strip())
|
|
9875
|
+
return 1
|
|
9876
|
+
|
|
9877
|
+
# A Findings block normally holds several findings with different
|
|
9878
|
+
# dispositions, written as free prose on one line or wrapped across
|
|
9879
|
+
# several — `review-entry-extent` covers the wrapped shape. The
|
|
9880
|
+
# evidence taken from a marker is what follows *that* marker, not the
|
|
9881
|
+
# rest of the text — otherwise the first `Resolved here:` swallows
|
|
9882
|
+
# every disposition after it, and a block mixing a fix with a
|
|
9883
|
+
# tracker-owned follow-up is refused on the follow-up's URL. Measured
|
|
9884
|
+
# on this change's own task 1.3 before it was fixed.
|
|
9885
|
+
mixed = complete(
|
|
9886
|
+
"the counter's own line arithmetic was wrong. Resolved here: M1. "
|
|
9887
|
+
"Second, nothing warned that the CLI was four minors old. "
|
|
9888
|
+
f"Durable owner: {TRACKER_OWNER}"
|
|
9889
|
+
)
|
|
9890
|
+
if mixed.returncode != 0:
|
|
9891
|
+
report(
|
|
9892
|
+
"findings-resolved-here: M1 a block holding a resolved finding "
|
|
9893
|
+
"and a tracker-owned one was refused; the resolved marker must "
|
|
9894
|
+
"not reach past its own evidence."
|
|
9895
|
+
)
|
|
9896
|
+
report((mixed.stderr or mixed.stdout).strip())
|
|
9897
|
+
return 1
|
|
9898
|
+
|
|
9899
|
+
second_bare = complete(
|
|
9900
|
+
"the first was fixed. Resolved here: M1. The second was too. "
|
|
9901
|
+
"Resolved here:"
|
|
9902
|
+
)
|
|
9903
|
+
if second_bare.returncode != 3:
|
|
9904
|
+
report(
|
|
9905
|
+
"findings-resolved-here: M2 every resolved marker must be "
|
|
9906
|
+
"checked, not only the first; a block whose second names no "
|
|
9907
|
+
f"evidence exited {second_bare.returncode}, not 3."
|
|
9908
|
+
)
|
|
9909
|
+
report((second_bare.stderr or second_bare.stdout).strip())
|
|
9910
|
+
return 1
|
|
9911
|
+
if not problems(second_bare, "finding-resolution-evidence"):
|
|
9912
|
+
report(
|
|
9913
|
+
"findings-resolved-here: M2 the second resolved marker was not "
|
|
9914
|
+
"reported as missing its evidence."
|
|
9915
|
+
)
|
|
9916
|
+
return 1
|
|
9917
|
+
|
|
9918
|
+
# M3 — the general refusal has to name the form that now exists.
|
|
9919
|
+
unowned = complete("a finding with no disposition at all")
|
|
9920
|
+
owner_message = problems(unowned, "finding-owner")
|
|
9921
|
+
if unowned.returncode != 3:
|
|
9922
|
+
report(
|
|
9923
|
+
"findings-resolved-here: M3 a finding with no disposition "
|
|
9924
|
+
f"exited {unowned.returncode}, not 3."
|
|
9925
|
+
)
|
|
9926
|
+
report((unowned.stderr or unowned.stdout).strip())
|
|
9927
|
+
return 1
|
|
9928
|
+
for phrase in ("Resolved here", "M<n>", "Discard reason", "Durable owner"):
|
|
9929
|
+
if phrase not in owner_message:
|
|
9930
|
+
report(
|
|
9931
|
+
"findings-resolved-here: M3 the accepted-forms message must "
|
|
9932
|
+
f"name every disposition; {phrase!r} is missing from it."
|
|
9933
|
+
)
|
|
9934
|
+
report(owner_message or "(no finding-owner problem)")
|
|
9935
|
+
return 1
|
|
9936
|
+
|
|
9937
|
+
# M4 — the criterion is stated where the author reads it, and the portable
|
|
9938
|
+
# skill and its projection do not disagree about it.
|
|
9939
|
+
canonical = ROOT / "src/skills/keel-review-checklist/SKILL.md"
|
|
9940
|
+
projected = ROOT / "plugins/keel/skills/keel-review-checklist/SKILL.md"
|
|
9941
|
+
if canonical.read_bytes() != projected.read_bytes():
|
|
9942
|
+
report(
|
|
9943
|
+
"findings-resolved-here: M4 the portable checklist and its plugin "
|
|
9944
|
+
"projection are not byte-identical."
|
|
9945
|
+
)
|
|
9946
|
+
return 1
|
|
9947
|
+
checklist = canonical.read_text(encoding="utf-8")
|
|
9948
|
+
for phrase in ("Resolved here:", "Durable owner:", "Discard reason:"):
|
|
9949
|
+
if phrase not in checklist:
|
|
9950
|
+
report(
|
|
9951
|
+
"findings-resolved-here: M4 keel-review-checklist does not name "
|
|
9952
|
+
f"the {phrase!r} disposition."
|
|
9953
|
+
)
|
|
9954
|
+
return 1
|
|
9955
|
+
agents = (ROOT / "AGENTS.md").read_text(encoding="utf-8")
|
|
9956
|
+
if "Resolved here:" not in agents:
|
|
9957
|
+
report(
|
|
9958
|
+
"findings-resolved-here: M4 AGENTS.md does not name the "
|
|
9959
|
+
"`Resolved here:` disposition."
|
|
9960
|
+
)
|
|
9961
|
+
return 1
|
|
9962
|
+
|
|
9963
|
+
if "findings-resolved-here" not in {name for name, _ in SCENARIOS}:
|
|
9964
|
+
report("findings-resolved-here: the scenario registry does not include it.")
|
|
9965
|
+
return 1
|
|
9966
|
+
report("findings-resolved-here scenario passed.")
|
|
9967
|
+
return 0
|
|
9968
|
+
|
|
9969
|
+
|
|
9970
|
+
# A task whose Review entries are supplied verbatim, so a fixture can wrap one
|
|
9971
|
+
# across lines. Everything outside the Review block is the complete, checked
|
|
9972
|
+
# shape `tracker_owner_tasks` uses; only the four entries and the sibling below
|
|
9973
|
+
# them vary.
|
|
9974
|
+
def review_extent_tasks(review: str, blocker: str = "none") -> str:
|
|
9975
|
+
return (
|
|
9976
|
+
"# Tasks\n\n## Invalidates\n\n- None.\n\n"
|
|
9977
|
+
"## Expectation Coverage\n\n"
|
|
9978
|
+
"- E1:\n"
|
|
9979
|
+
" - Covered by: 1.1\n\n"
|
|
9980
|
+
"## 1. Work\n\n"
|
|
9981
|
+
"- [x] 1.1 Complete behavior\n"
|
|
9982
|
+
" - Owner: keel-agent\n"
|
|
9983
|
+
" - Mode: implementation\n"
|
|
9984
|
+
" - Covers:\n"
|
|
9985
|
+
" - E1: public behavior\n"
|
|
9986
|
+
" - Read:\n"
|
|
9987
|
+
" - README.md\n"
|
|
9988
|
+
" - Touch:\n"
|
|
9989
|
+
" - src/feature.js\n"
|
|
9990
|
+
" - Commands:\n"
|
|
9991
|
+
" - M1: node test.js\n"
|
|
9992
|
+
" - Acceptance:\n"
|
|
9993
|
+
" - Public behavior passes.\n"
|
|
9994
|
+
" - Autonomy boundary:\n"
|
|
9995
|
+
" - Default: hard-stop\n"
|
|
9996
|
+
" - Pre-authorized fallback: none\n"
|
|
9997
|
+
" - Coupling: none\n"
|
|
9998
|
+
" - Candidate Boundary:\n"
|
|
9999
|
+
" - One candidate.\n"
|
|
10000
|
+
" - Stop Rules:\n"
|
|
10001
|
+
" - Stop on failure.\n"
|
|
10002
|
+
" - Evidence:\n"
|
|
10003
|
+
" - Contract: pending\n"
|
|
10004
|
+
" - M1: passed\n"
|
|
10005
|
+
" - Review:\n"
|
|
10006
|
+
f"{review}"
|
|
10007
|
+
f" - Blocker: {blocker}\n"
|
|
10008
|
+
" - Stop if:\n"
|
|
10009
|
+
" - Scope expands.\n"
|
|
10010
|
+
" - Report:\n"
|
|
10011
|
+
" - Summary\n"
|
|
10012
|
+
)
|
|
10013
|
+
|
|
10014
|
+
|
|
10015
|
+
def validate_review_entry_extent_scenario() -> int:
|
|
10016
|
+
"""A Review entry is the text the author wrote under its label.
|
|
10017
|
+
|
|
10018
|
+
Issue #49's first supplement. `reviewValue()` matched each Review entry
|
|
10019
|
+
with a line-anchored regex, so an entry that wrapped was judged by its
|
|
10020
|
+
first line and every continuation line was discarded before any check saw
|
|
10021
|
+
it. A `Findings` recording its `Durable owner:` on the fourth line was
|
|
10022
|
+
refused with the owner present, the path existing, and the form correct.
|
|
10023
|
+
|
|
10024
|
+
The same truncation failed open, which #49 does not report: a `Findings`
|
|
10025
|
+
reading `none` on its first line and recording an unowned finding below it
|
|
10026
|
+
produced no problem at all, because the check tested `none` and never saw
|
|
10027
|
+
the rest.
|
|
10028
|
+
|
|
10029
|
+
Both directions are driven through `keel gate task-complete` on real
|
|
10030
|
+
repositories, and the widened read is bounded at the sibling entry so that
|
|
10031
|
+
one entry cannot be satisfied by another's text.
|
|
10032
|
+
"""
|
|
10033
|
+
label = "review-entry-extent"
|
|
10034
|
+
owner_path = "openspec/changes/demo/tasks.md"
|
|
10035
|
+
|
|
10036
|
+
with tempfile.TemporaryDirectory(
|
|
10037
|
+
prefix="keel-review-extent-", ignore_cleanup_errors=True
|
|
10038
|
+
) as raw:
|
|
10039
|
+
repo = Path(raw)
|
|
10040
|
+
tasks = repo / "openspec/changes/demo/tasks.md"
|
|
10041
|
+
write_text(repo / "openspec/changes/demo/proposal.md", "# Proposal\n")
|
|
10042
|
+
write_text(
|
|
10043
|
+
repo / "openspec/changes/demo/specs/demo/spec.md",
|
|
10044
|
+
"## ADDED Requirements\n",
|
|
10045
|
+
)
|
|
10046
|
+
|
|
10047
|
+
def complete(review: str, blocker: str = "none") -> dict | None:
|
|
10048
|
+
write_text(tasks, review_extent_tasks(review, blocker))
|
|
10049
|
+
if not record_contract_anchor(repo, "demo"):
|
|
10050
|
+
return None
|
|
10051
|
+
result = run_keel(
|
|
10052
|
+
repo, "gate", "task-complete", ".",
|
|
10053
|
+
"--change", "demo", "--task", "1.1", "--json",
|
|
10054
|
+
)
|
|
10055
|
+
return json.loads(result.stdout) if result.stdout.strip() else {}
|
|
10056
|
+
|
|
10057
|
+
def codes(payload: dict) -> list[str]:
|
|
10058
|
+
return sorted(
|
|
10059
|
+
item.get("code", "") for item in payload.get("problems", [])
|
|
10060
|
+
)
|
|
10061
|
+
|
|
10062
|
+
def owner_problems(payload: dict) -> list[str]:
|
|
10063
|
+
return [
|
|
10064
|
+
item.get("message", "")
|
|
10065
|
+
for item in payload.get("problems", [])
|
|
10066
|
+
if item.get("code") == "finding-owner"
|
|
10067
|
+
]
|
|
10068
|
+
|
|
10069
|
+
def readable(payload: dict | None, fixture: str) -> bool:
|
|
10070
|
+
if payload is None:
|
|
10071
|
+
report(
|
|
10072
|
+
f"{label} could not record a Contract anchor for {fixture}: "
|
|
10073
|
+
"task-start refused the fixture, so task-complete never ran "
|
|
10074
|
+
"against it."
|
|
10075
|
+
)
|
|
10076
|
+
return False
|
|
10077
|
+
if not payload:
|
|
10078
|
+
report(
|
|
10079
|
+
f"{label} got no JSON from task-complete for {fixture}, so "
|
|
10080
|
+
"there was no verdict to read rather than a verdict that "
|
|
10081
|
+
"was wrong."
|
|
10082
|
+
)
|
|
10083
|
+
return False
|
|
10084
|
+
return True
|
|
10085
|
+
|
|
10086
|
+
def review_block(findings: str, status: str = "pass") -> str:
|
|
10087
|
+
return (
|
|
10088
|
+
f" - Status: {status}\n"
|
|
10089
|
+
" - Acceptance check: reviewed\n"
|
|
10090
|
+
" - Scope check: reviewed\n"
|
|
10091
|
+
f"{findings}"
|
|
10092
|
+
)
|
|
10093
|
+
|
|
10094
|
+
# M1 — the reported defect. The same finding is written twice: wrapped
|
|
10095
|
+
# across four indented lines with the owner on the last, and joined
|
|
10096
|
+
# onto one line with no word changed. The pair is compared as well as
|
|
10097
|
+
# pinned absolutely, because a comparison alone passes when both forms
|
|
10098
|
+
# are refused identically.
|
|
10099
|
+
wrapped_owner = (
|
|
10100
|
+
" - Findings: two behaviors of state sync turned up in "
|
|
10101
|
+
"measurement.\n"
|
|
10102
|
+
" First, the watcher fires per property rather than once "
|
|
10103
|
+
"per object.\n"
|
|
10104
|
+
" Second, a class-typed property does not update on the "
|
|
10105
|
+
"child side.\n"
|
|
10106
|
+
f" Durable owner: {owner_path}\n"
|
|
10107
|
+
)
|
|
10108
|
+
joined_owner = (
|
|
10109
|
+
" - Findings: two behaviors of state sync turned up in "
|
|
10110
|
+
"measurement. First, the watcher fires per property rather than "
|
|
10111
|
+
"once per object. Second, a class-typed property does not update "
|
|
10112
|
+
f"on the child side. Durable owner: {owner_path}\n"
|
|
10113
|
+
)
|
|
10114
|
+
forms = {
|
|
10115
|
+
"wrapped": complete(review_block(wrapped_owner)),
|
|
10116
|
+
"joined": complete(review_block(joined_owner)),
|
|
10117
|
+
}
|
|
10118
|
+
for form, payload in forms.items():
|
|
10119
|
+
if not readable(payload, f"owner-{form}"):
|
|
10120
|
+
return 1
|
|
10121
|
+
stray = owner_problems(payload)
|
|
10122
|
+
if stray:
|
|
10123
|
+
report(
|
|
10124
|
+
f"{label} refused a durable owner the author recorded, with "
|
|
10125
|
+
f"the finding written {form}. The owner is named after "
|
|
10126
|
+
"`Durable owner:` and the path exists."
|
|
10127
|
+
)
|
|
10128
|
+
report(repr(stray))
|
|
10129
|
+
return 1
|
|
10130
|
+
if codes(forms["wrapped"]) != codes(forms["joined"]):
|
|
10131
|
+
report(
|
|
10132
|
+
f"{label} returned different problems for the same finding "
|
|
10133
|
+
"depending on whether it wrapped."
|
|
10134
|
+
)
|
|
10135
|
+
report(f"wrapped={codes(forms['wrapped'])} joined={codes(forms['joined'])}")
|
|
10136
|
+
return 1
|
|
10137
|
+
|
|
10138
|
+
# M2 — the silent half. `none` on the first line, a real finding under
|
|
10139
|
+
# it. At 5.27.0 this produced no problem at all.
|
|
10140
|
+
below_none = complete(
|
|
10141
|
+
review_block(
|
|
10142
|
+
" - Findings: none\n"
|
|
10143
|
+
" Actually the retry path still drops the last error.\n"
|
|
10144
|
+
" Nobody owns this and no reason is given for dropping "
|
|
10145
|
+
"it.\n"
|
|
10146
|
+
)
|
|
10147
|
+
)
|
|
10148
|
+
if not readable(below_none, "finding-below-none"):
|
|
10149
|
+
return 1
|
|
10150
|
+
if not owner_problems(below_none):
|
|
10151
|
+
report(
|
|
10152
|
+
f"{label} accepted a finding written below a `none` first "
|
|
10153
|
+
"line. The gate read the word `none` and not the finding under "
|
|
10154
|
+
"it."
|
|
10155
|
+
)
|
|
10156
|
+
report(repr(codes(below_none)))
|
|
10157
|
+
return 1
|
|
10158
|
+
|
|
10159
|
+
# M3 — the sibling bound. The wrapped `Findings` carries no disposition
|
|
10160
|
+
# of its own; the `- Blocker:` entry below it does. Reading the entry
|
|
10161
|
+
# whole must not let the sibling's text satisfy it.
|
|
10162
|
+
absorbing = complete(
|
|
10163
|
+
review_block(
|
|
10164
|
+
" - Findings: the helper rewrote its own baseline path.\n"
|
|
10165
|
+
" Nothing here names who owns it.\n"
|
|
10166
|
+
),
|
|
10167
|
+
blocker=f"none. Durable owner: {owner_path}",
|
|
10168
|
+
)
|
|
10169
|
+
if not readable(absorbing, "sibling-bound"):
|
|
10170
|
+
return 1
|
|
10171
|
+
if not owner_problems(absorbing):
|
|
10172
|
+
report(
|
|
10173
|
+
f"{label} let a `Findings` with no disposition be satisfied by "
|
|
10174
|
+
"the sibling entry below it. A widened read must still stop at "
|
|
10175
|
+
"its sibling."
|
|
10176
|
+
)
|
|
10177
|
+
report(repr(codes(absorbing)))
|
|
10178
|
+
return 1
|
|
10179
|
+
|
|
10180
|
+
# M4 — the unwrapped shape, which is what all 640 archived Review
|
|
10181
|
+
# entries use. The expected message is written out rather than compared
|
|
10182
|
+
# against another run of the same build, so a change that altered the
|
|
10183
|
+
# text in both places would still fail here.
|
|
10184
|
+
unwrapped = complete(
|
|
10185
|
+
review_block(" - Findings: a finding with no disposition\n")
|
|
10186
|
+
)
|
|
10187
|
+
if not readable(unwrapped, "unwrapped"):
|
|
10188
|
+
return 1
|
|
10189
|
+
unwrapped_owner = owner_problems(unwrapped)
|
|
10190
|
+
if len(unwrapped_owner) != 1:
|
|
10191
|
+
report(
|
|
10192
|
+
f"{label} did not produce exactly one `finding-owner` problem "
|
|
10193
|
+
"for an unwrapped finding with no disposition."
|
|
10194
|
+
)
|
|
10195
|
+
report(repr(codes(unwrapped)))
|
|
10196
|
+
return 1
|
|
10197
|
+
for phrase in (
|
|
10198
|
+
"Review Findings must be `none` or carry a disposition — name a "
|
|
10199
|
+
"path after `Durable owner:`",
|
|
10200
|
+
"Resolved here:",
|
|
10201
|
+
"Durable owner:",
|
|
10202
|
+
"Discard reason:",
|
|
10203
|
+
"keel/HANDOFF.md",
|
|
10204
|
+
):
|
|
10205
|
+
if phrase not in unwrapped_owner[0]:
|
|
10206
|
+
report(
|
|
10207
|
+
f"{label} the unwrapped refusal's message changed; "
|
|
10208
|
+
f"{phrase!r} is no longer in it. This change moves how much "
|
|
10209
|
+
"text is read, not what the gate says."
|
|
10210
|
+
)
|
|
10211
|
+
report(unwrapped_owner[0])
|
|
10212
|
+
return 1
|
|
10213
|
+
passing = complete(
|
|
10214
|
+
review_block(f" - Findings: still open. Durable owner: {owner_path}\n")
|
|
10215
|
+
)
|
|
10216
|
+
if not readable(passing, "unwrapped-owned"):
|
|
10217
|
+
return 1
|
|
10218
|
+
if owner_problems(passing):
|
|
10219
|
+
report(
|
|
10220
|
+
f"{label} refused an unwrapped finding that names an existing "
|
|
10221
|
+
"path after `Durable owner:`."
|
|
10222
|
+
)
|
|
10223
|
+
report(repr(owner_problems(passing)))
|
|
10224
|
+
return 1
|
|
10225
|
+
|
|
10226
|
+
if "review-entry-extent" not in {name for name, _ in SCENARIOS}:
|
|
10227
|
+
report(f"{label}: the scenario registry does not include it.")
|
|
10228
|
+
return 1
|
|
10229
|
+
report(f"{label} scenario passed.")
|
|
10230
|
+
return 0
|
|
10231
|
+
|
|
10232
|
+
|
|
10233
|
+
# A task whose Reauthorizations entry is supplied verbatim, so a fixture can
|
|
10234
|
+
# leave it out entirely, wrap it, or plant an unfilled token on either the
|
|
10235
|
+
# label line or a continuation line. Everything else is the complete, checked
|
|
10236
|
+
# shape `review_extent_tasks` uses; only the Reauthorizations line (or its
|
|
10237
|
+
# absence) and Blocker vary.
|
|
10238
|
+
def reauthorizations_tasks(reauthorizations: str, blocker: str = "none") -> str:
|
|
10239
|
+
return (
|
|
10240
|
+
"# Tasks\n\n## Invalidates\n\n- None.\n\n"
|
|
10241
|
+
"## Expectation Coverage\n\n"
|
|
10242
|
+
"- E1:\n"
|
|
10243
|
+
" - Covered by: 1.1\n\n"
|
|
10244
|
+
"## 1. Work\n\n"
|
|
10245
|
+
"- [x] 1.1 Complete behavior\n"
|
|
10246
|
+
" - Owner: keel-agent\n"
|
|
10247
|
+
" - Mode: implementation\n"
|
|
10248
|
+
" - Covers:\n"
|
|
10249
|
+
" - E1: public behavior\n"
|
|
10250
|
+
" - Read:\n"
|
|
10251
|
+
" - README.md\n"
|
|
10252
|
+
" - Touch:\n"
|
|
10253
|
+
" - src/feature.js\n"
|
|
10254
|
+
" - Commands:\n"
|
|
10255
|
+
" - M1: node test.js\n"
|
|
10256
|
+
" - Acceptance:\n"
|
|
10257
|
+
" - Public behavior passes.\n"
|
|
10258
|
+
" - Autonomy boundary:\n"
|
|
10259
|
+
" - Default: hard-stop\n"
|
|
10260
|
+
" - Pre-authorized fallback: none\n"
|
|
10261
|
+
" - Coupling: none\n"
|
|
10262
|
+
" - Candidate Boundary:\n"
|
|
10263
|
+
" - One candidate.\n"
|
|
10264
|
+
" - Stop Rules:\n"
|
|
10265
|
+
" - Stop on failure.\n"
|
|
10266
|
+
" - Evidence:\n"
|
|
10267
|
+
" - Contract: pending\n"
|
|
10268
|
+
" - M1: passed\n"
|
|
10269
|
+
" - Review:\n"
|
|
10270
|
+
" - Status: pass\n"
|
|
10271
|
+
" - Acceptance check: reviewed\n"
|
|
10272
|
+
" - Scope check: reviewed\n"
|
|
10273
|
+
" - Findings: none\n"
|
|
10274
|
+
f" - Blocker: {blocker}\n"
|
|
10275
|
+
f"{reauthorizations}"
|
|
10276
|
+
" - Stop if:\n"
|
|
10277
|
+
" - Scope expands.\n"
|
|
10278
|
+
" - Report:\n"
|
|
10279
|
+
" - Summary\n"
|
|
10280
|
+
)
|
|
10281
|
+
|
|
10282
|
+
|
|
10283
|
+
def validate_reauthorizations_shape_scenario() -> int:
|
|
10284
|
+
"""A rejection becomes a Reauthorizations entry the author writes (#70).
|
|
10285
|
+
|
|
10286
|
+
The owner chose R2 over R3 on issue #70: the task author records what a
|
|
10287
|
+
gate rejected themselves, in a new `Reauthorizations` Evidence entry; the
|
|
10288
|
+
gate adds no write of its own and validates only the entry's shape. This
|
|
10289
|
+
drives that shape check through `keel gate task-complete` on real
|
|
10290
|
+
repositories: absence and `none` need nothing, a concrete record — wrapped
|
|
10291
|
+
or joined — is accepted, an unfilled `<slot>` token is refused even when it
|
|
10292
|
+
only appears on a continuation line the label line itself does not carry,
|
|
10293
|
+
and a concrete record never blocks completion the way a concrete `Blocker`
|
|
10294
|
+
does.
|
|
10295
|
+
"""
|
|
10296
|
+
label = "reauthorizations-shape"
|
|
10297
|
+
baseline_reauthorizations = " - Reauthorizations: none\n"
|
|
10298
|
+
baseline_blocker = " - Blocker: none\n"
|
|
10299
|
+
|
|
10300
|
+
with tempfile.TemporaryDirectory(
|
|
10301
|
+
prefix="keel-reauthorizations-shape-", ignore_cleanup_errors=True
|
|
10302
|
+
) as raw:
|
|
10303
|
+
repo = Path(raw)
|
|
10304
|
+
tasks = repo / "openspec/changes/demo/tasks.md"
|
|
10305
|
+
write_text(repo / "openspec/changes/demo/proposal.md", "# Proposal\n")
|
|
10306
|
+
write_text(
|
|
10307
|
+
repo / "openspec/changes/demo/specs/demo/spec.md",
|
|
10308
|
+
"## ADDED Requirements\n",
|
|
10309
|
+
)
|
|
10310
|
+
|
|
10311
|
+
# Started once, with clean Evidence, and every variant below edits
|
|
10312
|
+
# only the Reauthorizations/Blocker lines and calls task-complete
|
|
10313
|
+
# directly rather than restarting. Two things make restarting wrong
|
|
10314
|
+
# here: Evidence is not part of the compiled capsule that task-start
|
|
10315
|
+
# fingerprints, so editing only these lines cannot drift it; and
|
|
10316
|
+
# task-start separately enforces its own blanket "Evidence must be
|
|
10317
|
+
# concrete" check across the whole field regardless of which label an
|
|
10318
|
+
# unfilled `<slot>` sits under, so planting one for M1 and then
|
|
10319
|
+
# restarting would be refused there before task-complete's more
|
|
10320
|
+
# specific `reauthorizations-shape` ever saw it.
|
|
10321
|
+
write_text(tasks, reauthorizations_tasks(baseline_reauthorizations))
|
|
10322
|
+
if not record_contract_anchor(repo, "demo"):
|
|
10323
|
+
report(f"{label} could not record a Contract anchor for the baseline fixture.")
|
|
10324
|
+
return 1
|
|
10325
|
+
baseline = tasks.read_text(encoding="utf-8")
|
|
10326
|
+
if baseline_reauthorizations not in baseline or baseline_blocker not in baseline:
|
|
10327
|
+
report(
|
|
10328
|
+
f"{label} lost track of the baseline Reauthorizations/Blocker "
|
|
10329
|
+
"lines after task-start recorded the Contract anchor."
|
|
10330
|
+
)
|
|
10331
|
+
return 1
|
|
10332
|
+
|
|
10333
|
+
def complete(reauthorizations: str, blocker: str = "none") -> dict:
|
|
10334
|
+
text = baseline.replace(baseline_reauthorizations, reauthorizations, 1)
|
|
10335
|
+
if blocker != "none":
|
|
10336
|
+
text = text.replace(baseline_blocker, f" - Blocker: {blocker}\n", 1)
|
|
10337
|
+
write_text(tasks, text)
|
|
10338
|
+
result = run_keel(
|
|
10339
|
+
repo, "gate", "task-complete", ".",
|
|
10340
|
+
"--change", "demo", "--task", "1.1", "--json",
|
|
10341
|
+
)
|
|
10342
|
+
return json.loads(result.stdout) if result.stdout.strip() else {}
|
|
10343
|
+
|
|
10344
|
+
def codes(payload: dict) -> list[str]:
|
|
10345
|
+
return sorted(
|
|
10346
|
+
item.get("code", "") for item in payload.get("problems", [])
|
|
10347
|
+
)
|
|
10348
|
+
|
|
10349
|
+
def shape_problems(payload: dict) -> list[str]:
|
|
10350
|
+
return [
|
|
10351
|
+
item.get("message", "")
|
|
10352
|
+
for item in payload.get("problems", [])
|
|
10353
|
+
if item.get("code") == "reauthorizations-shape"
|
|
10354
|
+
]
|
|
10355
|
+
|
|
10356
|
+
def readable(payload: dict, fixture: str) -> bool:
|
|
10357
|
+
if not payload:
|
|
10358
|
+
report(
|
|
10359
|
+
f"{label} got no JSON from task-complete for {fixture}, so "
|
|
10360
|
+
"there was no verdict to read rather than a verdict that "
|
|
10361
|
+
"was wrong."
|
|
10362
|
+
)
|
|
10363
|
+
return False
|
|
10364
|
+
return True
|
|
10365
|
+
|
|
10366
|
+
# M1 — the core new behavior, in both places an unfilled token can
|
|
10367
|
+
# hide: on the label line itself, and on a continuation line a
|
|
10368
|
+
# first-line-only reader would never see.
|
|
10369
|
+
on_label_line = complete(" - Reauthorizations: <what was rejected>\n")
|
|
10370
|
+
if not readable(on_label_line, "unfilled-on-label-line"):
|
|
10371
|
+
return 1
|
|
10372
|
+
if not shape_problems(on_label_line):
|
|
10373
|
+
report(
|
|
10374
|
+
f"{label} accepted an unfilled `<slot>` token written on the "
|
|
10375
|
+
"Reauthorizations label line itself."
|
|
10376
|
+
)
|
|
10377
|
+
report(repr(codes(on_label_line)))
|
|
10378
|
+
return 1
|
|
10379
|
+
if "<what was rejected>" not in shape_problems(on_label_line)[0]:
|
|
10380
|
+
report(f"{label} refused the unfilled token but did not name it.")
|
|
10381
|
+
report(shape_problems(on_label_line)[0])
|
|
10382
|
+
return 1
|
|
10383
|
+
|
|
10384
|
+
on_continuation = complete(
|
|
10385
|
+
" - Reauthorizations:\n"
|
|
10386
|
+
" - task-complete refused finding-owner on 2026-08-05; added\n"
|
|
10387
|
+
" a Durable owner naming <the tracker issue>.\n"
|
|
8609
10388
|
)
|
|
8610
|
-
if
|
|
10389
|
+
if not readable(on_continuation, "unfilled-on-continuation-line"):
|
|
10390
|
+
return 1
|
|
10391
|
+
if not shape_problems(on_continuation):
|
|
8611
10392
|
report(
|
|
8612
|
-
"
|
|
8613
|
-
"
|
|
10393
|
+
f"{label} accepted an unfilled `<slot>` token written on a "
|
|
10394
|
+
"continuation line below the Reauthorizations label. A reader "
|
|
10395
|
+
"that only looks at the first line would miss it."
|
|
8614
10396
|
)
|
|
8615
|
-
report((
|
|
10397
|
+
report(repr(codes(on_continuation)))
|
|
10398
|
+
return 1
|
|
10399
|
+
if "<the tracker issue>" not in shape_problems(on_continuation)[0]:
|
|
10400
|
+
report(
|
|
10401
|
+
f"{label} refused the continuation-line token but did not "
|
|
10402
|
+
"name it."
|
|
10403
|
+
)
|
|
10404
|
+
report(shape_problems(on_continuation)[0])
|
|
8616
10405
|
return 1
|
|
8617
10406
|
|
|
8618
|
-
#
|
|
8619
|
-
#
|
|
8620
|
-
#
|
|
8621
|
-
|
|
8622
|
-
|
|
8623
|
-
|
|
8624
|
-
|
|
8625
|
-
mixed = complete(
|
|
8626
|
-
"the counter's own line arithmetic was wrong. Resolved here: M1. "
|
|
8627
|
-
"Second, nothing warned that the CLI was four minors old. "
|
|
8628
|
-
f"Durable owner: {TRACKER_OWNER}"
|
|
8629
|
-
)
|
|
8630
|
-
if mixed.returncode != 0:
|
|
10407
|
+
# M2 (regression) — no false positives: absent, none, and a wrapped
|
|
10408
|
+
# concrete record all pass, and wrapped/joined forms of the same
|
|
10409
|
+
# concrete text agree.
|
|
10410
|
+
absent = complete("")
|
|
10411
|
+
if not readable(absent, "absent"):
|
|
10412
|
+
return 1
|
|
10413
|
+
if shape_problems(absent):
|
|
8631
10414
|
report(
|
|
8632
|
-
"
|
|
8633
|
-
"
|
|
8634
|
-
"not reach past its own evidence."
|
|
10415
|
+
f"{label} refused a task that declares no Reauthorizations "
|
|
10416
|
+
"entry at all."
|
|
8635
10417
|
)
|
|
8636
|
-
report((
|
|
10418
|
+
report(repr(shape_problems(absent)))
|
|
8637
10419
|
return 1
|
|
8638
10420
|
|
|
8639
|
-
|
|
8640
|
-
|
|
8641
|
-
|
|
10421
|
+
bare_none = complete(" - Reauthorizations: none\n")
|
|
10422
|
+
if not readable(bare_none, "bare-none"):
|
|
10423
|
+
return 1
|
|
10424
|
+
if shape_problems(bare_none):
|
|
10425
|
+
report(f"{label} refused a bare `Reauthorizations: none`.")
|
|
10426
|
+
report(repr(shape_problems(bare_none)))
|
|
10427
|
+
return 1
|
|
10428
|
+
|
|
10429
|
+
wrapped_concrete = complete(
|
|
10430
|
+
" - Reauthorizations:\n"
|
|
10431
|
+
" - task-complete refused outside-touch on src/helper.js on\n"
|
|
10432
|
+
" 2026-08-05; the file belonged to this task and Touch was\n"
|
|
10433
|
+
" missing it, so it was added.\n"
|
|
8642
10434
|
)
|
|
8643
|
-
|
|
10435
|
+
joined_concrete = complete(
|
|
10436
|
+
" - Reauthorizations: task-complete refused outside-touch on "
|
|
10437
|
+
"src/helper.js on 2026-08-05; the file belonged to this task and "
|
|
10438
|
+
"Touch was missing it, so it was added.\n"
|
|
10439
|
+
)
|
|
10440
|
+
if not readable(wrapped_concrete, "wrapped-concrete"):
|
|
10441
|
+
return 1
|
|
10442
|
+
if not readable(joined_concrete, "joined-concrete"):
|
|
10443
|
+
return 1
|
|
10444
|
+
if shape_problems(wrapped_concrete):
|
|
10445
|
+
report(f"{label} refused a concrete, wrapped Reauthorizations record.")
|
|
10446
|
+
report(repr(shape_problems(wrapped_concrete)))
|
|
10447
|
+
return 1
|
|
10448
|
+
if codes(wrapped_concrete) != codes(joined_concrete):
|
|
8644
10449
|
report(
|
|
8645
|
-
"
|
|
8646
|
-
"
|
|
8647
|
-
f"evidence exited {second_bare.returncode}, not 3."
|
|
10450
|
+
f"{label} returned different problems for the same concrete "
|
|
10451
|
+
"record depending on whether it wrapped."
|
|
8648
10452
|
)
|
|
8649
|
-
report((second_bare.stderr or second_bare.stdout).strip())
|
|
8650
|
-
return 1
|
|
8651
|
-
if not problems(second_bare, "finding-resolution-evidence"):
|
|
8652
10453
|
report(
|
|
8653
|
-
"
|
|
8654
|
-
"reported as missing its evidence."
|
|
10454
|
+
f"wrapped={codes(wrapped_concrete)} joined={codes(joined_concrete)}"
|
|
8655
10455
|
)
|
|
8656
10456
|
return 1
|
|
8657
10457
|
|
|
8658
|
-
# M3 —
|
|
8659
|
-
|
|
8660
|
-
|
|
8661
|
-
if
|
|
10458
|
+
# M3 (regression) — presence does not block completion, unlike
|
|
10459
|
+
# Blocker. Proven able to fail: a concrete Blocker on the same
|
|
10460
|
+
# repository does fail it, so the comparison is not vacuous.
|
|
10461
|
+
if codes(wrapped_concrete):
|
|
8662
10462
|
report(
|
|
8663
|
-
"
|
|
8664
|
-
|
|
10463
|
+
f"{label} a concrete Reauthorizations record produced a "
|
|
10464
|
+
"problem — presence alone must not fail completion."
|
|
8665
10465
|
)
|
|
8666
|
-
report((
|
|
10466
|
+
report(repr(codes(wrapped_concrete)))
|
|
8667
10467
|
return 1
|
|
8668
|
-
|
|
8669
|
-
|
|
8670
|
-
|
|
8671
|
-
|
|
8672
|
-
f"name every disposition; {phrase!r} is missing from it."
|
|
8673
|
-
)
|
|
8674
|
-
report(owner_message or "(no finding-owner problem)")
|
|
8675
|
-
return 1
|
|
8676
|
-
|
|
8677
|
-
# M4 — the criterion is stated where the author reads it, and the portable
|
|
8678
|
-
# skill and its projection do not disagree about it.
|
|
8679
|
-
canonical = ROOT / "src/skills/keel-review-checklist/SKILL.md"
|
|
8680
|
-
projected = ROOT / "plugins/keel/skills/keel-review-checklist/SKILL.md"
|
|
8681
|
-
if canonical.read_bytes() != projected.read_bytes():
|
|
8682
|
-
report(
|
|
8683
|
-
"findings-resolved-here: M4 the portable checklist and its plugin "
|
|
8684
|
-
"projection are not byte-identical."
|
|
8685
|
-
)
|
|
8686
|
-
return 1
|
|
8687
|
-
checklist = canonical.read_text(encoding="utf-8")
|
|
8688
|
-
for phrase in ("Resolved here:", "Durable owner:", "Discard reason:"):
|
|
8689
|
-
if phrase not in checklist:
|
|
10468
|
+
concrete_blocker = complete("", blocker="a real blocker is recorded here")
|
|
10469
|
+
if not readable(concrete_blocker, "concrete-blocker"):
|
|
10470
|
+
return 1
|
|
10471
|
+
if "blocker" not in codes(concrete_blocker):
|
|
8690
10472
|
report(
|
|
8691
|
-
"
|
|
8692
|
-
|
|
10473
|
+
f"{label} the comparison fixture proves nothing: a concrete "
|
|
10474
|
+
"Blocker on this repository did not fail completion either."
|
|
8693
10475
|
)
|
|
10476
|
+
report(repr(codes(concrete_blocker)))
|
|
8694
10477
|
return 1
|
|
8695
|
-
agents = (ROOT / "AGENTS.md").read_text(encoding="utf-8")
|
|
8696
|
-
if "Resolved here:" not in agents:
|
|
8697
|
-
report(
|
|
8698
|
-
"findings-resolved-here: M4 AGENTS.md does not name the "
|
|
8699
|
-
"`Resolved here:` disposition."
|
|
8700
|
-
)
|
|
8701
|
-
return 1
|
|
8702
10478
|
|
|
8703
|
-
if
|
|
8704
|
-
report("
|
|
10479
|
+
if label not in {name for name, _ in SCENARIOS}:
|
|
10480
|
+
report(f"{label}: the scenario registry does not include it.")
|
|
8705
10481
|
return 1
|
|
8706
|
-
report("
|
|
10482
|
+
report(f"{label} scenario passed.")
|
|
8707
10483
|
return 0
|
|
8708
10484
|
|
|
8709
10485
|
|
|
@@ -8900,6 +10676,38 @@ def validate_guard_status_is_not_enforcement_scenario() -> int:
|
|
|
8900
10676
|
return 0
|
|
8901
10677
|
|
|
8902
10678
|
|
|
10679
|
+
def validate_guard_warnings_are_concise_scenario() -> int:
|
|
10680
|
+
"""Issue #92 item 1: shorten the two standing guard warnings.
|
|
10681
|
+
|
|
10682
|
+
The owner authorized shortening wording, not removing content: every
|
|
10683
|
+
idea `guard-status-is-not-enforcement` checks for must still resolve, but
|
|
10684
|
+
the combined text must drop below the byte counts `#92` measured before
|
|
10685
|
+
this fix (398 for `guard status`, 397 for `guard clear`, on a fresh
|
|
10686
|
+
directory with no manifest).
|
|
10687
|
+
"""
|
|
10688
|
+
baseline = {"status": 398, "clear": 397}
|
|
10689
|
+
with tempfile.TemporaryDirectory(prefix="keel-guard-concise-") as raw:
|
|
10690
|
+
project = Path(raw)
|
|
10691
|
+
for label, before in baseline.items():
|
|
10692
|
+
result = run_keel(project, "guard", label)
|
|
10693
|
+
out = result.stdout or ""
|
|
10694
|
+
if len(out) >= before:
|
|
10695
|
+
report(
|
|
10696
|
+
f"guard-warnings-are-concise: keel guard {label} is "
|
|
10697
|
+
f"{len(out)} chars, not shorter than the {before}-char "
|
|
10698
|
+
"baseline #92 measured before the wording was shortened."
|
|
10699
|
+
)
|
|
10700
|
+
return 1
|
|
10701
|
+
if "guard-warnings-are-concise" not in {name for name, _ in SCENARIOS}:
|
|
10702
|
+
report(
|
|
10703
|
+
"guard-warnings-are-concise: the scenario registry does not "
|
|
10704
|
+
"include it."
|
|
10705
|
+
)
|
|
10706
|
+
return 1
|
|
10707
|
+
report("guard-warnings-are-concise scenario passed.")
|
|
10708
|
+
return 0
|
|
10709
|
+
|
|
10710
|
+
|
|
8903
10711
|
def validate_source_repo_cli_resolution_scenario() -> int:
|
|
8904
10712
|
"""Issue #13 item 3: a bare `keel` runs the installed package.
|
|
8905
10713
|
|
|
@@ -9839,6 +11647,38 @@ def validate_core_gates_scenario() -> int:
|
|
|
9839
11647
|
report((handoff_owner.stderr or handoff_owner.stdout).strip())
|
|
9840
11648
|
return 1
|
|
9841
11649
|
|
|
11650
|
+
# The accepted-forms enumeration is a menu; the one actionable
|
|
11651
|
+
# instruction inside it (name a path after `Durable owner:`) must lead
|
|
11652
|
+
# the message, not close it, so a reader who forgot to name a path
|
|
11653
|
+
# reads what to do before reading the menu. Three single-cause checks,
|
|
11654
|
+
# not one compound condition — issue #43 exists to catch exactly that
|
|
11655
|
+
# shape.
|
|
11656
|
+
finding_owner_lower = finding_owner_message.lower()
|
|
11657
|
+
instruction_at = finding_owner_lower.find(
|
|
11658
|
+
"name a path after `durable owner:`"
|
|
11659
|
+
)
|
|
11660
|
+
if instruction_at < 0:
|
|
11661
|
+
report(
|
|
11662
|
+
"core-gates scenario: the finding-owner message dropped its "
|
|
11663
|
+
"actionable instruction (\"name a path after `Durable "
|
|
11664
|
+
"owner:`\")."
|
|
11665
|
+
)
|
|
11666
|
+
return 1
|
|
11667
|
+
resolved_here_at = finding_owner_lower.find("`resolved here:`")
|
|
11668
|
+
if resolved_here_at < 0:
|
|
11669
|
+
report(
|
|
11670
|
+
"core-gates scenario: the finding-owner message dropped its "
|
|
11671
|
+
"`Resolved here:` disposition form."
|
|
11672
|
+
)
|
|
11673
|
+
return 1
|
|
11674
|
+
if instruction_at > resolved_here_at:
|
|
11675
|
+
report(
|
|
11676
|
+
"core-gates scenario: the finding-owner message states its "
|
|
11677
|
+
"actionable instruction after the `Resolved here:` form "
|
|
11678
|
+
"instead of before it."
|
|
11679
|
+
)
|
|
11680
|
+
return 1
|
|
11681
|
+
|
|
9842
11682
|
write_text(
|
|
9843
11683
|
completion_tasks,
|
|
9844
11684
|
completion_task(
|
|
@@ -13835,6 +15675,99 @@ def validate_standing_authorization_declaration_scenario() -> int:
|
|
|
13835
15675
|
return 0
|
|
13836
15676
|
|
|
13837
15677
|
|
|
15678
|
+
def validate_standing_authorization_sync_confusion_scenario() -> int:
|
|
15679
|
+
"""`sync` is a `change-close --action` value, not an `authorize:` name (#93).
|
|
15680
|
+
|
|
15681
|
+
A reader who copies `sync` from `change-close --action sync|archive` into
|
|
15682
|
+
`authorize:` gets the generic unrecognized-action message, which never says
|
|
15683
|
+
why `sync` in particular is wrong or what to write instead — and the
|
|
15684
|
+
failure was reported only by `keel --doctor`, an explicitly-invoked
|
|
15685
|
+
diagnostic, not by `keel context`, the command a session runs first.
|
|
15686
|
+
"""
|
|
15687
|
+
label = "standing-authorization-sync-confusion"
|
|
15688
|
+
with tempfile.TemporaryDirectory(prefix="keel-authorize-sync-") as raw_tmp:
|
|
15689
|
+
root = Path(raw_tmp)
|
|
15690
|
+
|
|
15691
|
+
# M1 — `sync` gets a sentence naming the `change-close --action`
|
|
15692
|
+
# confusion and pointing at `archive`; an unrelated typo does not.
|
|
15693
|
+
sync_repo = root / "sync"
|
|
15694
|
+
sync_repo.mkdir()
|
|
15695
|
+
write_authorize_config(sync_repo, "authorize:\n - sync\n")
|
|
15696
|
+
doctor = run_keel(sync_repo, "--doctor")
|
|
15697
|
+
combined = doctor.stdout + doctor.stderr
|
|
15698
|
+
for needle in ("change-close --action", "declare `archive`"):
|
|
15699
|
+
if needle not in combined:
|
|
15700
|
+
report(
|
|
15701
|
+
f"{label} M1 the doctor message does not name the sync "
|
|
15702
|
+
f"confusion: {needle!r}."
|
|
15703
|
+
)
|
|
15704
|
+
report(combined)
|
|
15705
|
+
return 1
|
|
15706
|
+
|
|
15707
|
+
typo_repo = root / "typo"
|
|
15708
|
+
typo_repo.mkdir()
|
|
15709
|
+
write_authorize_config(typo_repo, "authorize:\n - deploy\n")
|
|
15710
|
+
typo_doctor = run_keel(typo_repo, "--doctor")
|
|
15711
|
+
typo_combined = typo_doctor.stdout + typo_doctor.stderr
|
|
15712
|
+
if "deploy" not in typo_combined:
|
|
15713
|
+
report(f"{label} M1 the typo repo's own offending entry went unreported.")
|
|
15714
|
+
report(typo_combined)
|
|
15715
|
+
return 1
|
|
15716
|
+
if "change-close --action" in typo_combined:
|
|
15717
|
+
report(
|
|
15718
|
+
f"{label} M1 a genuine typo (`deploy`) still received the "
|
|
15719
|
+
"sync-specific sentence."
|
|
15720
|
+
)
|
|
15721
|
+
report(typo_combined)
|
|
15722
|
+
return 1
|
|
15723
|
+
|
|
15724
|
+
# M2 — the same broken declaration surfaces in `keel context`'s
|
|
15725
|
+
# warnings, and does not disturb status or nextAction, exactly as an
|
|
15726
|
+
# uncommitted git path is reported without changing selection.
|
|
15727
|
+
context = json.loads(run_keel(sync_repo, "context", "--json").stdout)
|
|
15728
|
+
warnings = " ".join(context.get("warnings", []))
|
|
15729
|
+
if "change-close --action" not in warnings or "sync" not in warnings:
|
|
15730
|
+
report(
|
|
15731
|
+
f"{label} M2 `keel context` did not report the broken "
|
|
15732
|
+
"authorize: declaration."
|
|
15733
|
+
)
|
|
15734
|
+
report(f" warnings: {context.get('warnings')!r}")
|
|
15735
|
+
return 1
|
|
15736
|
+
if context.get("status") != "idle":
|
|
15737
|
+
report(f"{label} M2 the broken declaration changed context's status: {context.get('status')!r}")
|
|
15738
|
+
return 1
|
|
15739
|
+
if context.get("nextAction", {}).get("kind") != "none":
|
|
15740
|
+
report(
|
|
15741
|
+
"{} M2 the broken declaration changed context's nextAction: {!r}".format(
|
|
15742
|
+
label, context.get("nextAction")
|
|
15743
|
+
)
|
|
15744
|
+
)
|
|
15745
|
+
return 1
|
|
15746
|
+
|
|
15747
|
+
typo_context = json.loads(run_keel(typo_repo, "context", "--json").stdout)
|
|
15748
|
+
typo_warnings = " ".join(typo_context.get("warnings", []))
|
|
15749
|
+
if "deploy" not in typo_warnings:
|
|
15750
|
+
report(
|
|
15751
|
+
f"{label} M2 `keel context` did not report the typo repo's "
|
|
15752
|
+
"broken authorize: declaration at all."
|
|
15753
|
+
)
|
|
15754
|
+
report(f" warnings: {typo_context.get('warnings')!r}")
|
|
15755
|
+
return 1
|
|
15756
|
+
if "change-close --action" in typo_warnings:
|
|
15757
|
+
report(
|
|
15758
|
+
f"{label} M2 a genuine typo (`deploy`) still received the "
|
|
15759
|
+
"sync-specific sentence inside `keel context`'s warning."
|
|
15760
|
+
)
|
|
15761
|
+
report(f" warnings: {typo_context.get('warnings')!r}")
|
|
15762
|
+
return 1
|
|
15763
|
+
|
|
15764
|
+
if label not in {name for name, _ in SCENARIOS}:
|
|
15765
|
+
report(f"{label}: the scenario registry does not include it.")
|
|
15766
|
+
return 1
|
|
15767
|
+
report(f"{label} scenario passed.")
|
|
15768
|
+
return 0
|
|
15769
|
+
|
|
15770
|
+
|
|
13838
15771
|
def standing_authorization_task(boundary: str = "") -> str:
|
|
13839
15772
|
return (
|
|
13840
15773
|
"- [ ] 1.1 Behavior\n"
|
|
@@ -14237,7 +16170,274 @@ def validate_triage_declaration_scenario() -> int:
|
|
|
14237
16170
|
report(f"triage: the same inputs gave different answers: {offline} != {again}")
|
|
14238
16171
|
return 1
|
|
14239
16172
|
|
|
14240
|
-
report("triage-declaration scenario passed.")
|
|
16173
|
+
report("triage-declaration scenario passed.")
|
|
16174
|
+
return 0
|
|
16175
|
+
|
|
16176
|
+
|
|
16177
|
+
def validate_triage_admits_from_the_repository_scenario() -> int:
|
|
16178
|
+
"""The owner's decision belongs in the owner's file, not on the reporter's issue.
|
|
16179
|
+
|
|
16180
|
+
A label records "this may run unattended" in a field the person who filed
|
|
16181
|
+
the issue can see, in the vocabulary they were asked to file under. The
|
|
16182
|
+
second source moves that decision into keel/config.yaml without giving it
|
|
16183
|
+
up to inference: a number is applied by hand to one issue, exactly as a
|
|
16184
|
+
label is. Reported as #62.
|
|
16185
|
+
"""
|
|
16186
|
+
|
|
16187
|
+
def declare(repo: Path, body: str) -> None:
|
|
16188
|
+
(repo / "keel").mkdir(parents=True, exist_ok=True)
|
|
16189
|
+
(repo / "keel" / "config.yaml").write_text(
|
|
16190
|
+
f"fast_check: echo check\n{body}", encoding="utf-8"
|
|
16191
|
+
)
|
|
16192
|
+
|
|
16193
|
+
def triage(repo: Path, *args: str) -> dict | None:
|
|
16194
|
+
result = run_keel(repo, "triage", ".", *args, "--json")
|
|
16195
|
+
try:
|
|
16196
|
+
return json.loads(result.stdout)
|
|
16197
|
+
except json.JSONDecodeError:
|
|
16198
|
+
# Reported here rather than at the call site. "did not admit" and
|
|
16199
|
+
# "did not run" are two different diagnoses, and a caller checking
|
|
16200
|
+
# `verdict is None or verdict["status"] != "admit"` would send the
|
|
16201
|
+
# reader looking for a verdict in a run that produced none.
|
|
16202
|
+
report(
|
|
16203
|
+
f"keel triage {' '.join(args)} printed no JSON "
|
|
16204
|
+
f"(exit {result.returncode}): "
|
|
16205
|
+
f"{(result.stderr or result.stdout).strip()}"
|
|
16206
|
+
)
|
|
16207
|
+
return None
|
|
16208
|
+
|
|
16209
|
+
def make(root: Path, name: str, body: str) -> Path:
|
|
16210
|
+
repo = root / name
|
|
16211
|
+
repo.mkdir()
|
|
16212
|
+
declare(repo, body)
|
|
16213
|
+
return repo
|
|
16214
|
+
|
|
16215
|
+
nested = "triage:\n labels:\n - auto\n issues:\n - 62\n"
|
|
16216
|
+
|
|
16217
|
+
with tempfile.TemporaryDirectory(prefix="keel-triage-repo-") as raw_tmp:
|
|
16218
|
+
root = Path(raw_tmp)
|
|
16219
|
+
|
|
16220
|
+
# M1 — a number listed in the repository's own file admits, and says so.
|
|
16221
|
+
numbers = make(root, "numbers", "triage:\n issues:\n - 62\n - 58\n")
|
|
16222
|
+
admitted = triage(numbers, "--issue", "62", "--labels", "bug")
|
|
16223
|
+
if admitted is None or admitted.get("status") != "admit":
|
|
16224
|
+
report(f"M1 a declared issue number did not admit: {admitted}")
|
|
16225
|
+
return 1
|
|
16226
|
+
reason = admitted.get("reason") or ""
|
|
16227
|
+
if "62" not in reason or "keel/config.yaml" not in reason:
|
|
16228
|
+
report(
|
|
16229
|
+
"M1 the admission does not name the number and the file that "
|
|
16230
|
+
f"lists it, so a reader cannot tell which source answered: {reason}"
|
|
16231
|
+
)
|
|
16232
|
+
return 1
|
|
16233
|
+
if admitted.get("sources") != ["issue"]:
|
|
16234
|
+
report(f"M1 the admitting source is not reported as the issue: {admitted}")
|
|
16235
|
+
return 1
|
|
16236
|
+
refused = triage(numbers, "--issue", "61", "--labels", "bug")
|
|
16237
|
+
if refused is None or refused.get("status") != "refuse":
|
|
16238
|
+
report(f"M1 an unlisted number was admitted: {refused}")
|
|
16239
|
+
return 1
|
|
16240
|
+
|
|
16241
|
+
# M2 — the flat form is what every installed repository has. Its
|
|
16242
|
+
# entries keep their exact meaning, down to the reason text, and none
|
|
16243
|
+
# of them is read as a number.
|
|
16244
|
+
flat = make(root, "flat", "triage:\n - auto\n")
|
|
16245
|
+
legacy_admit = triage(flat, "--labels", "auto,bug")
|
|
16246
|
+
if legacy_admit is None or legacy_admit.get("status") != "admit":
|
|
16247
|
+
report(f"M2 the bare list stopped admitting its labels: {legacy_admit}")
|
|
16248
|
+
return 1
|
|
16249
|
+
expected_admit = (
|
|
16250
|
+
"admitted by declared label auto; admission starts work and decides "
|
|
16251
|
+
"nothing after it — every later gate still applies and a material "
|
|
16252
|
+
"decision still stops for the owner."
|
|
16253
|
+
)
|
|
16254
|
+
if (legacy_admit.get("reason") or "") != expected_admit:
|
|
16255
|
+
report(
|
|
16256
|
+
"M2 the bare list's admission reason changed from the 5.23.0 "
|
|
16257
|
+
f"text: {legacy_admit.get('reason')!r}"
|
|
16258
|
+
)
|
|
16259
|
+
return 1
|
|
16260
|
+
if legacy_admit.get("sources") != ["label"]:
|
|
16261
|
+
report(f"M2 the bare list's admitting source is not the label: {legacy_admit}")
|
|
16262
|
+
return 1
|
|
16263
|
+
legacy_refuse = triage(flat, "--labels", "bug")
|
|
16264
|
+
expected_refuse = "the issue carries bug and this repository accepts auto."
|
|
16265
|
+
if legacy_refuse is None or legacy_refuse.get("status") != "refuse":
|
|
16266
|
+
report(f"M2 the bare list admitted an undeclared label: {legacy_refuse}")
|
|
16267
|
+
return 1
|
|
16268
|
+
if (legacy_refuse.get("reason") or "") != expected_refuse:
|
|
16269
|
+
report(
|
|
16270
|
+
"M2 the bare list's refusal reason changed from the 5.23.0 "
|
|
16271
|
+
f"text: {legacy_refuse.get('reason')!r}"
|
|
16272
|
+
)
|
|
16273
|
+
return 1
|
|
16274
|
+
# The entry is the label `auto`, not the number 62 — a bare token must
|
|
16275
|
+
# not have been reclassified, because that would move an authorization
|
|
16276
|
+
# boundary in a repository nobody edited.
|
|
16277
|
+
by_number = triage(flat, "--issue", "62", "--labels", "bug")
|
|
16278
|
+
if by_number is None or by_number.get("status") != "refuse":
|
|
16279
|
+
report(f"M2 a bare list entry was read as an issue number: {by_number}")
|
|
16280
|
+
return 1
|
|
16281
|
+
|
|
16282
|
+
# M2 — and either source admits alone when both are declared.
|
|
16283
|
+
both = make(root, "both", nested)
|
|
16284
|
+
for args, source in (
|
|
16285
|
+
(("--labels", "auto"), "label"),
|
|
16286
|
+
(("--issue", "62"), "issue"),
|
|
16287
|
+
):
|
|
16288
|
+
verdict = triage(both, *args)
|
|
16289
|
+
if verdict is None or verdict.get("status") != "admit":
|
|
16290
|
+
report(f"M2 the {source} source did not admit alone: {verdict}")
|
|
16291
|
+
return 1
|
|
16292
|
+
neither = triage(both, "--labels", "bug", "--issue", "7")
|
|
16293
|
+
if neither is None or neither.get("status") != "refuse":
|
|
16294
|
+
report(f"M2 an issue matching neither source was admitted: {neither}")
|
|
16295
|
+
return 1
|
|
16296
|
+
neither_reason = neither.get("reason") or ""
|
|
16297
|
+
for needle in ("bug", "7", "auto", "62"):
|
|
16298
|
+
if needle not in neither_reason:
|
|
16299
|
+
report(
|
|
16300
|
+
"M2 the refusal must name what the issue carried and both "
|
|
16301
|
+
f"halves of what is accepted; missing {needle}: {neither_reason}"
|
|
16302
|
+
)
|
|
16303
|
+
return 1
|
|
16304
|
+
|
|
16305
|
+
# M3 — a declaration Keel cannot fully read admits nothing, including
|
|
16306
|
+
# the half it did read, and names the part that failed.
|
|
16307
|
+
unreadable = {
|
|
16308
|
+
"hash": ("triage:\n issues:\n - '#62'\n labels:\n - auto\n", "#62"),
|
|
16309
|
+
"unknown-key": (
|
|
16310
|
+
"triage:\n authors:\n - someone\n labels:\n - auto\n",
|
|
16311
|
+
"authors",
|
|
16312
|
+
),
|
|
16313
|
+
"mixed": ("triage:\n - auto\n issues:\n - 62\n", "auto"),
|
|
16314
|
+
"inline": ("triage: { issues: [62] }\n", "issues"),
|
|
16315
|
+
}
|
|
16316
|
+
undeclared = make(root, "undeclared", "")
|
|
16317
|
+
undeclared_reason = (triage(undeclared, "--labels", "auto") or {}).get(
|
|
16318
|
+
"reason"
|
|
16319
|
+
) or ""
|
|
16320
|
+
for name, (body, needle) in unreadable.items():
|
|
16321
|
+
repo = make(root, name, body)
|
|
16322
|
+
# Every fixture above carries something Keel *did* read — the label
|
|
16323
|
+
# `auto`, or the number 62. A partial read would admit here.
|
|
16324
|
+
verdict = triage(repo, "--labels", "auto", "--issue", "62")
|
|
16325
|
+
if verdict is None or verdict.get("status") != "refuse":
|
|
16326
|
+
report(
|
|
16327
|
+
f"M3 the {name} declaration admitted on the half Keel could "
|
|
16328
|
+
f"read: {verdict}"
|
|
16329
|
+
)
|
|
16330
|
+
return 1
|
|
16331
|
+
reason = verdict.get("reason") or ""
|
|
16332
|
+
if needle not in reason:
|
|
16333
|
+
report(
|
|
16334
|
+
f"M3 the {name} refusal does not name the entry it could not "
|
|
16335
|
+
f"read ({needle}): {reason}"
|
|
16336
|
+
)
|
|
16337
|
+
return 1
|
|
16338
|
+
if reason == undeclared_reason:
|
|
16339
|
+
report(
|
|
16340
|
+
f"M3 the {name} refusal is word for word the undeclared-policy "
|
|
16341
|
+
"refusal, so a broken declaration reads as an absent one."
|
|
16342
|
+
)
|
|
16343
|
+
return 1
|
|
16344
|
+
if "could not" not in reason.lower():
|
|
16345
|
+
report(f"M3 the {name} refusal does not say it could not read: {reason}")
|
|
16346
|
+
return 1
|
|
16347
|
+
|
|
16348
|
+
# M3 — and the CLI refuses the same spelling it refuses in the file,
|
|
16349
|
+
# and refuses to answer with no issue attributes at all.
|
|
16350
|
+
hashed = run_keel(numbers, "triage", ".", "--issue", "#62")
|
|
16351
|
+
if hashed.returncode == 0:
|
|
16352
|
+
report("M3 --issue accepted a number Keel refuses in the declaration.")
|
|
16353
|
+
return 1
|
|
16354
|
+
empty = run_keel(numbers, "triage", ".")
|
|
16355
|
+
if empty.returncode == 0:
|
|
16356
|
+
report("M3 triage answered with neither labels nor an issue number.")
|
|
16357
|
+
return 1
|
|
16358
|
+
|
|
16359
|
+
# M4 — what may start work here is answerable from one command.
|
|
16360
|
+
doctor_both = run_keel(both, "--doctor").stdout
|
|
16361
|
+
if "Unattended triage:" not in doctor_both or "triage: ok" not in doctor_both:
|
|
16362
|
+
report(f"M4 doctor does not report a declared triage surface:\n{doctor_both}")
|
|
16363
|
+
return 1
|
|
16364
|
+
for needle in ("auto", "62"):
|
|
16365
|
+
if needle not in doctor_both:
|
|
16366
|
+
report(
|
|
16367
|
+
f"M4 doctor omits the declared {needle} source, so the surface "
|
|
16368
|
+
f"under-reports what may start work:\n{doctor_both}"
|
|
16369
|
+
)
|
|
16370
|
+
return 1
|
|
16371
|
+
doctor_numbers = run_keel(numbers, "--doctor").stdout
|
|
16372
|
+
if "triage: ok" not in doctor_numbers or "62" not in doctor_numbers:
|
|
16373
|
+
report(f"M4 doctor does not report an issues-only policy:\n{doctor_numbers}")
|
|
16374
|
+
return 1
|
|
16375
|
+
if "labelled" in doctor_numbers:
|
|
16376
|
+
report(
|
|
16377
|
+
"M4 doctor names labels for a repository that declared none:\n"
|
|
16378
|
+
f"{doctor_numbers}"
|
|
16379
|
+
)
|
|
16380
|
+
return 1
|
|
16381
|
+
doctor_broken = run_keel(root / "hash", "--doctor").stdout
|
|
16382
|
+
if "triage: ok" in doctor_broken:
|
|
16383
|
+
report(
|
|
16384
|
+
"M4 doctor reports an unreadable declaration as a working policy:\n"
|
|
16385
|
+
f"{doctor_broken}"
|
|
16386
|
+
)
|
|
16387
|
+
return 1
|
|
16388
|
+
if "#62" not in doctor_broken:
|
|
16389
|
+
report(
|
|
16390
|
+
"M4 doctor does not name what it could not read, so the owner "
|
|
16391
|
+
f"cannot tell a broken declaration from an absent one:\n{doctor_broken}"
|
|
16392
|
+
)
|
|
16393
|
+
return 1
|
|
16394
|
+
|
|
16395
|
+
# M5 — the surfaces an owner reads must state the second source. Phrases,
|
|
16396
|
+
# not keywords: "issue number" appearing anywhere would satisfy a keyword
|
|
16397
|
+
# check while the page still teaches that a label is the only unit. The
|
|
16398
|
+
# alignment skill points at the protocol's own statement instead of
|
|
16399
|
+
# repeating it (keel-unattended-triage's "A secondary surface points
|
|
16400
|
+
# instead of repeating" scenario), so only the three full surfaces below
|
|
16401
|
+
# are held to the phrase itself; the skill copies are held to the pointer.
|
|
16402
|
+
full_surfaces = {
|
|
16403
|
+
"config": ROOT / "keel/config.yaml",
|
|
16404
|
+
"protocol": ROOT / "AGENTS.md",
|
|
16405
|
+
"readme": ROOT / "README.md",
|
|
16406
|
+
}
|
|
16407
|
+
stale = "A label is the unit"
|
|
16408
|
+
for label, path in full_surfaces.items():
|
|
16409
|
+
if not path.is_file():
|
|
16410
|
+
report(f"M5 missing surface: {path}")
|
|
16411
|
+
return 1
|
|
16412
|
+
content = re.sub(r"\s+", " ", path.read_text(encoding="utf-8"))
|
|
16413
|
+
if "issue number" not in content:
|
|
16414
|
+
report(f"M5 {label} does not state the issue-number source: {path}")
|
|
16415
|
+
return 1
|
|
16416
|
+
if stale in content:
|
|
16417
|
+
report(
|
|
16418
|
+
f"M5 {label} still says {stale!r}, which stopped being true when "
|
|
16419
|
+
f"a second source was declared: {path}"
|
|
16420
|
+
)
|
|
16421
|
+
return 1
|
|
16422
|
+
|
|
16423
|
+
canonical = ROOT / "src/skills/keel-align-expectations/SKILL.md"
|
|
16424
|
+
distributed = ROOT / PLUGIN_ROOT / "skills/keel-align-expectations/SKILL.md"
|
|
16425
|
+
for label, path in (("canonical skill", canonical), ("distributed skill", distributed)):
|
|
16426
|
+
if not path.is_file():
|
|
16427
|
+
report(f"M5 missing surface: {path}")
|
|
16428
|
+
return 1
|
|
16429
|
+
content = re.sub(r"\s+", " ", path.read_text(encoding="utf-8"))
|
|
16430
|
+
if "AGENTS.md" not in content or "states no separate copy" not in content:
|
|
16431
|
+
report(
|
|
16432
|
+
f"M5 {label} does not point to the protocol's issue-number "
|
|
16433
|
+
f"statement: {path}"
|
|
16434
|
+
)
|
|
16435
|
+
return 1
|
|
16436
|
+
if canonical.read_bytes() != distributed.read_bytes():
|
|
16437
|
+
report("M5 the canonical and distributed skills diverged.")
|
|
16438
|
+
return 1
|
|
16439
|
+
|
|
16440
|
+
report("triage-admits-from-the-repository scenario passed.")
|
|
14241
16441
|
return 0
|
|
14242
16442
|
|
|
14243
16443
|
|
|
@@ -15338,7 +17538,11 @@ def validate_unattended_boundary_scenario() -> int:
|
|
|
15338
17538
|
"""The boundary must be readable where an unattended run will read it.
|
|
15339
17539
|
|
|
15340
17540
|
Phrases, not keywords: "unattended" appearing somewhere would satisfy a
|
|
15341
|
-
keyword check while stating none of what a run may and may not do.
|
|
17541
|
+
keyword check while stating none of what a run may and may not do. The
|
|
17542
|
+
alignment skill points at the protocol's own statement instead of
|
|
17543
|
+
repeating it (keel-unattended-triage's "A secondary surface points
|
|
17544
|
+
instead of repeating" scenario), so only the protocol is held to the
|
|
17545
|
+
full phrase set; the skill copies are held to the pointer instead.
|
|
15342
17546
|
"""
|
|
15343
17547
|
|
|
15344
17548
|
required = [
|
|
@@ -15352,22 +17556,35 @@ def validate_unattended_boundary_scenario() -> int:
|
|
|
15352
17556
|
# Admission comes from a declaration, never from accumulated history.
|
|
15353
17557
|
"never from a precedent",
|
|
15354
17558
|
]
|
|
17559
|
+
pointer_required = [
|
|
17560
|
+
"AGENTS.md",
|
|
17561
|
+
"Unattended runs",
|
|
17562
|
+
"states no separate copy",
|
|
17563
|
+
]
|
|
15355
17564
|
canonical = ROOT / "src/skills/keel-align-expectations/SKILL.md"
|
|
15356
17565
|
distributed = ROOT / PLUGIN_ROOT / "skills/keel-align-expectations/SKILL.md"
|
|
15357
17566
|
protocol = ROOT / "AGENTS.md"
|
|
15358
17567
|
|
|
17568
|
+
if not protocol.is_file():
|
|
17569
|
+
report(f"unattended-boundary: missing protocol: {protocol}")
|
|
17570
|
+
return 1
|
|
17571
|
+
# Collapse whitespace: these are multi-word phrases in hard-wrapped
|
|
17572
|
+
# prose, so raw matching would assert the line layout, not the wording.
|
|
17573
|
+
protocol_content = re.sub(r"\s+", " ", protocol.read_text(encoding="utf-8"))
|
|
17574
|
+
for phrase in required:
|
|
17575
|
+
if phrase not in protocol_content:
|
|
17576
|
+
report(f"unattended-boundary: protocol omits: {phrase}")
|
|
17577
|
+
return 1
|
|
17578
|
+
|
|
15359
17579
|
for label, path in (
|
|
15360
|
-
("protocol", protocol),
|
|
15361
17580
|
("canonical skill", canonical),
|
|
15362
17581
|
("distributed skill", distributed),
|
|
15363
17582
|
):
|
|
15364
17583
|
if not path.is_file():
|
|
15365
17584
|
report(f"unattended-boundary: missing {label}: {path}")
|
|
15366
17585
|
return 1
|
|
15367
|
-
# Collapse whitespace: these are multi-word phrases in hard-wrapped
|
|
15368
|
-
# prose, so raw matching would assert the line layout, not the wording.
|
|
15369
17586
|
content = re.sub(r"\s+", " ", path.read_text(encoding="utf-8"))
|
|
15370
|
-
for phrase in
|
|
17587
|
+
for phrase in pointer_required:
|
|
15371
17588
|
if phrase not in content:
|
|
15372
17589
|
report(f"unattended-boundary: {label} omits: {phrase}")
|
|
15373
17590
|
return 1
|
|
@@ -19264,6 +21481,202 @@ def validate_guard_stale_manifest_scenario() -> int:
|
|
|
19264
21481
|
return 0
|
|
19265
21482
|
|
|
19266
21483
|
|
|
21484
|
+
# The message a reader is given when the state cannot be reauthorized.
|
|
21485
|
+
#
|
|
21486
|
+
# 5.26.0 recorded verbatim, so the parse-miss cell compares against the text as
|
|
21487
|
+
# it shipped rather than against whatever the code happens to produce.
|
|
21488
|
+
PARSE_MISS_PROBLEM = (
|
|
21489
|
+
"Guarded task demo#1.1 no longer resolves; reauthorize through "
|
|
21490
|
+
"`keel gate task-start` and `keel guard start`."
|
|
21491
|
+
)
|
|
21492
|
+
|
|
21493
|
+
|
|
21494
|
+
def guard_status_payload(repo: Path, cell: str) -> tuple[dict, str]:
|
|
21495
|
+
"""Return `guard status --json` as a payload plus its first problem message.
|
|
21496
|
+
|
|
21497
|
+
A run that produced nothing readable is reported here rather than handed
|
|
21498
|
+
back as an empty payload. Otherwise one condition downstream would guard two
|
|
21499
|
+
unrelated failures — the command did not run, and the status was wrong — and
|
|
21500
|
+
every caller would report the second, sending the reader to a state that was
|
|
21501
|
+
never observed. Returning `None` for the payload is what makes the two
|
|
21502
|
+
distinguishable at the call site instead of at the reader.
|
|
21503
|
+
"""
|
|
21504
|
+
result = run_keel(repo, "guard", "status", "--json")
|
|
21505
|
+
if not result.stdout.strip():
|
|
21506
|
+
report(
|
|
21507
|
+
f"guard-status-stale-manifest: {cell} `guard status --json` "
|
|
21508
|
+
f"produced no output (exit {result.returncode}). "
|
|
21509
|
+
f"{(result.stderr or '').strip()!r}"
|
|
21510
|
+
)
|
|
21511
|
+
return None, ""
|
|
21512
|
+
try:
|
|
21513
|
+
payload = json.loads(result.stdout)
|
|
21514
|
+
except json.JSONDecodeError as error:
|
|
21515
|
+
report(
|
|
21516
|
+
f"guard-status-stale-manifest: {cell} `guard status --json` "
|
|
21517
|
+
f"produced output that is not JSON ({error}): {result.stdout!r}"
|
|
21518
|
+
)
|
|
21519
|
+
return None, ""
|
|
21520
|
+
problems = payload.get("problems", [])
|
|
21521
|
+
return payload, (problems[0].get("message", "") if problems else "")
|
|
21522
|
+
|
|
21523
|
+
|
|
21524
|
+
def validate_guard_status_stale_manifest_scenario() -> int:
|
|
21525
|
+
"""`keel guard status` says which of two states the manifest is in.
|
|
21526
|
+
|
|
21527
|
+
`loadTaskContract` returns `null` when the tasks file is absent and when the
|
|
21528
|
+
task id is not in it. `guardStatus` had one branch for both, so a manifest
|
|
21529
|
+
whose change had been archived and a manifest whose task id was merely
|
|
21530
|
+
renumbered produced byte-identical output — and the shared message told the
|
|
21531
|
+
reader to reauthorize, which for the archived change is impossible:
|
|
21532
|
+
`keel gate task-start` reports a missing tasks file and `keel guard start`
|
|
21533
|
+
reports that the task does not exist. Neither names `keel guard clear`,
|
|
21534
|
+
which is the only action that resolves it.
|
|
21535
|
+
|
|
21536
|
+
The write guard hook drew this line in 5.12.0 by testing the change
|
|
21537
|
+
directory. This is the same test on the surface that describes the manifest
|
|
21538
|
+
rather than the one that refuses a write. Measured at the start of this
|
|
21539
|
+
change: both states returned `Guarded task demo#1.1 no longer resolves`.
|
|
21540
|
+
"""
|
|
21541
|
+
with tempfile.TemporaryDirectory(
|
|
21542
|
+
prefix="keel-guard-status-stale-", ignore_cleanup_errors=True
|
|
21543
|
+
) as raw_tmp:
|
|
21544
|
+
repo = Path(raw_tmp) / "repo"
|
|
21545
|
+
repo.mkdir()
|
|
21546
|
+
change_dir = repo / "openspec/changes/demo"
|
|
21547
|
+
write_text(change_dir / "tasks.md", guard_task_fixture())
|
|
21548
|
+
write_text(repo / "src/feature.js", "// fixture\n")
|
|
21549
|
+
|
|
21550
|
+
started = run_keel(
|
|
21551
|
+
repo, "guard", "start", "--change", "demo", "--task", "1.1", "--json"
|
|
21552
|
+
)
|
|
21553
|
+
if started.returncode != 0:
|
|
21554
|
+
report("guard-status-stale-manifest: guard start failed.")
|
|
21555
|
+
report((started.stderr or started.stdout).strip())
|
|
21556
|
+
return 1
|
|
21557
|
+
|
|
21558
|
+
# Relocated to where `openspec archive` moves it, for the reason the
|
|
21559
|
+
# guard-stale-manifest scenario records: what is under test is the
|
|
21560
|
+
# answer once the change is gone, not the archiver.
|
|
21561
|
+
archived_dir = repo / "openspec/changes/archive/2026-08-04-demo"
|
|
21562
|
+
archived_dir.parent.mkdir(parents=True, exist_ok=True)
|
|
21563
|
+
shutil.move(str(change_dir), str(archived_dir))
|
|
21564
|
+
|
|
21565
|
+
# M1 — the archived state names the directory that is gone and the
|
|
21566
|
+
# action that resolves it.
|
|
21567
|
+
stale, stale_message = guard_status_payload(repo, "M1")
|
|
21568
|
+
if stale is None:
|
|
21569
|
+
return 1
|
|
21570
|
+
if stale.get("status") != "drifted":
|
|
21571
|
+
report(
|
|
21572
|
+
"guard-status-stale-manifest: M1 expected the archived state to "
|
|
21573
|
+
f"stay classified as drifted, got {stale.get('status')!r}."
|
|
21574
|
+
)
|
|
21575
|
+
return 1
|
|
21576
|
+
stale_codes = [item.get("code") for item in stale.get("problems", [])]
|
|
21577
|
+
if "stale-manifest" not in stale_codes:
|
|
21578
|
+
report(
|
|
21579
|
+
"guard-status-stale-manifest: M1 an archived change was not "
|
|
21580
|
+
"reported under its own problem code, so a --json reader cannot "
|
|
21581
|
+
f"tell it from a parse miss. Codes: {stale_codes!r}"
|
|
21582
|
+
)
|
|
21583
|
+
return 1
|
|
21584
|
+
for needle in ("stale", "openspec/changes/demo", "keel guard clear"):
|
|
21585
|
+
if needle not in stale_message:
|
|
21586
|
+
report(
|
|
21587
|
+
"guard-status-stale-manifest: M1 the message must report the "
|
|
21588
|
+
f"manifest as stale and name what resolves it; {needle!r} is "
|
|
21589
|
+
f"missing from: {stale_message!r}"
|
|
21590
|
+
)
|
|
21591
|
+
return 1
|
|
21592
|
+
# The advice that could not be followed. `keel gate task-start` may
|
|
21593
|
+
# still be named for the task the reader is actually starting — what
|
|
21594
|
+
# must be gone is the instruction to reauthorize the vanished one.
|
|
21595
|
+
for forbidden in ("reauthorize through", "keel guard start"):
|
|
21596
|
+
if forbidden in stale_message:
|
|
21597
|
+
report(
|
|
21598
|
+
"guard-status-stale-manifest: M1 the message still sends the "
|
|
21599
|
+
f"reader to reauthorize a change that is not there; "
|
|
21600
|
+
f"{forbidden!r} is in: {stale_message!r}"
|
|
21601
|
+
)
|
|
21602
|
+
return 1
|
|
21603
|
+
|
|
21604
|
+
# M3 — the parse miss is untouched. A live change whose task id is
|
|
21605
|
+
# absent is not a fact about the repository, it is a read the guard
|
|
21606
|
+
# must not guess about, and its answer is the 5.26.0 answer verbatim.
|
|
21607
|
+
change_dir.mkdir(parents=True, exist_ok=True)
|
|
21608
|
+
write_text(
|
|
21609
|
+
change_dir / "tasks.md",
|
|
21610
|
+
guard_task_fixture().replace("1.1", "9.9"),
|
|
21611
|
+
)
|
|
21612
|
+
miss, miss_message = guard_status_payload(repo, "M3")
|
|
21613
|
+
if miss is None:
|
|
21614
|
+
return 1
|
|
21615
|
+
if miss.get("status") != "drifted":
|
|
21616
|
+
report(
|
|
21617
|
+
"guard-status-stale-manifest: M3 the parse miss changed status "
|
|
21618
|
+
f"from drifted to {miss.get('status')!r}."
|
|
21619
|
+
)
|
|
21620
|
+
return 1
|
|
21621
|
+
miss_codes = [item.get("code") for item in miss.get("problems", [])]
|
|
21622
|
+
if miss_codes != ["authority-drift"]:
|
|
21623
|
+
report(
|
|
21624
|
+
"guard-status-stale-manifest: M3 the parse miss changed problem "
|
|
21625
|
+
f"code. Expected ['authority-drift'], got {miss_codes!r}"
|
|
21626
|
+
)
|
|
21627
|
+
return 1
|
|
21628
|
+
if miss_message != PARSE_MISS_PROBLEM:
|
|
21629
|
+
report(
|
|
21630
|
+
"guard-status-stale-manifest: M3 the parse-miss message changed "
|
|
21631
|
+
f"from the 5.26.0 text.\nwas: {PARSE_MISS_PROBLEM!r}\nnow: "
|
|
21632
|
+
f"{miss_message!r}"
|
|
21633
|
+
)
|
|
21634
|
+
return 1
|
|
21635
|
+
|
|
21636
|
+
# M2 — the two states no longer answer the same. Asserted on both the
|
|
21637
|
+
# code and the message: a repair that reworded one branch while leaving
|
|
21638
|
+
# one code for two states would pass a message comparison alone.
|
|
21639
|
+
if stale_codes == miss_codes:
|
|
21640
|
+
report(
|
|
21641
|
+
"guard-status-stale-manifest: M2 an archived change and a task "
|
|
21642
|
+
"id absent from a live tasks.md still share a problem code "
|
|
21643
|
+
f"({stale_codes!r}), so they are separable only by prose."
|
|
21644
|
+
)
|
|
21645
|
+
return 1
|
|
21646
|
+
if stale_message == miss_message:
|
|
21647
|
+
report(
|
|
21648
|
+
"guard-status-stale-manifest: M2 the two states still produce "
|
|
21649
|
+
f"one message: {stale_message!r}"
|
|
21650
|
+
)
|
|
21651
|
+
return 1
|
|
21652
|
+
|
|
21653
|
+
# D4 — a third state: the change directory is there and its tasks file
|
|
21654
|
+
# is not. That is a live change mid-authoring, where reauthorizing after
|
|
21655
|
+
# writing the tasks file is genuinely the way out, so it keeps the
|
|
21656
|
+
# parse-miss message. This is what testing the directory rather than the
|
|
21657
|
+
# tasks file buys, and it is the cell that fails if the two are swapped.
|
|
21658
|
+
(change_dir / "tasks.md").unlink()
|
|
21659
|
+
authoring, authoring_message = guard_status_payload(repo, "D4")
|
|
21660
|
+
if authoring is None:
|
|
21661
|
+
return 1
|
|
21662
|
+
if authoring_message != PARSE_MISS_PROBLEM:
|
|
21663
|
+
report(
|
|
21664
|
+
"guard-status-stale-manifest: D4 a live change whose tasks.md "
|
|
21665
|
+
"is absent was reported as a stale manifest; the change is "
|
|
21666
|
+
f"still there and can be reauthorized. Got: {authoring_message!r}"
|
|
21667
|
+
)
|
|
21668
|
+
return 1
|
|
21669
|
+
|
|
21670
|
+
if "guard-status-stale-manifest" not in {name for name, _ in SCENARIOS}:
|
|
21671
|
+
report(
|
|
21672
|
+
"guard-status-stale-manifest: the scenario registry does not "
|
|
21673
|
+
"include it."
|
|
21674
|
+
)
|
|
21675
|
+
return 1
|
|
21676
|
+
report("guard-status-stale-manifest scenario passed.")
|
|
21677
|
+
return 0
|
|
21678
|
+
|
|
21679
|
+
|
|
19267
21680
|
# The one-message-many-failures shape, counted rather than forbidden.
|
|
19268
21681
|
#
|
|
19269
21682
|
# Issue #43 records it caught by `keel-review-checklist` in three consecutive
|
|
@@ -19287,7 +21700,15 @@ def validate_guard_stale_manifest_scenario() -> int:
|
|
|
19287
21700
|
# lowered, because a number that only checks for rises becomes false the first
|
|
19288
21701
|
# time someone fixes a site — and a false number is what this whole change is
|
|
19289
21702
|
# about. Fixing sites is expected. Lowering this constant is one line.
|
|
19290
|
-
|
|
21703
|
+
#
|
|
21704
|
+
# Raised 75 -> 80 by `change-verify-deferred-evidence` (issue #95): five sites,
|
|
21705
|
+
# each `payload.get("status") != "fail" or "<code>" not in codes(payload)`
|
|
21706
|
+
# guarding one `report(...)`, the same already-accepted gate-result-assertion
|
|
21707
|
+
# shape as `regression-check-tag`'s own site above. The message covers both
|
|
21708
|
+
# operands (the gate did not fail the way the fixture requires, whichever half
|
|
21709
|
+
# was wrong), and every site prints the full problem payload alongside it, so
|
|
21710
|
+
# a reader is never left with only the sentence.
|
|
21711
|
+
OR_GUARDED_ASSERTION_SITES = 80
|
|
19291
21712
|
|
|
19292
21713
|
|
|
19293
21714
|
def or_guarded_assertion_sites(source: str) -> list[int]:
|
|
@@ -19966,6 +22387,263 @@ def validate_decimal_runs_are_not_hash_shaped_scenario() -> int:
|
|
|
19966
22387
|
return 0
|
|
19967
22388
|
|
|
19968
22389
|
|
|
22390
|
+
def _tasks_semantics_probe(raw: str):
|
|
22391
|
+
"""Install into `raw` and return `check(body) -> (state, errors)`.
|
|
22392
|
+
|
|
22393
|
+
Both scenarios below ask the same question of `keel --check` — which lines
|
|
22394
|
+
of one active `tasks.md` it refuses — so they share the fixture rather than
|
|
22395
|
+
each growing their own copy of it. The caller owns the temporary directory,
|
|
22396
|
+
so a failed install is reported by the caller and cleaned up by its `with`.
|
|
22397
|
+
"""
|
|
22398
|
+
repo = (Path(raw) / "repo").resolve()
|
|
22399
|
+
repo.mkdir()
|
|
22400
|
+
install = run_keel(repo, "--install")
|
|
22401
|
+
if install.returncode != 0:
|
|
22402
|
+
return None, (install.stderr or install.stdout).strip()
|
|
22403
|
+
|
|
22404
|
+
tasks_path = repo / "openspec/changes/reading-lines/tasks.md"
|
|
22405
|
+
|
|
22406
|
+
def check(body: str):
|
|
22407
|
+
write_text(
|
|
22408
|
+
tasks_path,
|
|
22409
|
+
"# Tasks\n\n## Invalidates\n\n- None.\n\n"
|
|
22410
|
+
"## Tasks\n\n"
|
|
22411
|
+
f"{body}\n"
|
|
22412
|
+
"## Workflow Notes\n\n- None.\n",
|
|
22413
|
+
)
|
|
22414
|
+
result = run_keel(repo, "--check")
|
|
22415
|
+
if "keel state: ok" in result.stdout:
|
|
22416
|
+
state = "ok"
|
|
22417
|
+
elif "keel state: failed" in result.stdout:
|
|
22418
|
+
state = "failed"
|
|
22419
|
+
else:
|
|
22420
|
+
state = "unreported"
|
|
22421
|
+
errors = [
|
|
22422
|
+
out for out in result.stdout.splitlines() if out.startswith("state-error")
|
|
22423
|
+
]
|
|
22424
|
+
return state, errors
|
|
22425
|
+
|
|
22426
|
+
return check, None
|
|
22427
|
+
|
|
22428
|
+
|
|
22429
|
+
def validate_a_context_word_is_a_word_scenario() -> int:
|
|
22430
|
+
"""Issue #65 §1: `remaining` supplied the word `main`.
|
|
22431
|
+
|
|
22432
|
+
The context words that turn a hash-shaped token into a recorded commit
|
|
22433
|
+
identifier were matched anywhere on the line, so `remaining`, `heading`,
|
|
22434
|
+
`domain`, and `maintains` each supplied one. Four lines in this
|
|
22435
|
+
repository's own history are refused this way, and every hex token on
|
|
22436
|
+
them is a `sha256:` contract anchor rather than a commit identifier.
|
|
22437
|
+
|
|
22438
|
+
The boundary form is the owner's decision on #65 and keeps the inflections
|
|
22439
|
+
an author actually writes. Dropping `committed` and `hashes` would have
|
|
22440
|
+
been the quieter change and would have cost four real refusals.
|
|
22441
|
+
"""
|
|
22442
|
+
label = "a-context-word-is-a-word"
|
|
22443
|
+
with tempfile.TemporaryDirectory(prefix="keel-context-word-") as raw:
|
|
22444
|
+
check, failure = _tasks_semantics_probe(raw)
|
|
22445
|
+
if failure is not None:
|
|
22446
|
+
report(f"{label}: keel --install failed while building the fixture.")
|
|
22447
|
+
report(failure)
|
|
22448
|
+
return 1
|
|
22449
|
+
|
|
22450
|
+
# A word is not the words inside it. Each carries a hash-shaped token so
|
|
22451
|
+
# that the host word is the only thing under test.
|
|
22452
|
+
for host in ("remaining", "heading", "domain", "maintains"):
|
|
22453
|
+
state, errors = check(
|
|
22454
|
+
"- [x] A1 implementation\n"
|
|
22455
|
+
f" - Evidence:\n"
|
|
22456
|
+
f" - M1: pass — 0 anchors {host}, recorded at sha256:01b9e740ab.\n"
|
|
22457
|
+
)
|
|
22458
|
+
if state == "unreported":
|
|
22459
|
+
report(f"{label}: keel --check reported no state at all on `{host}`.")
|
|
22460
|
+
return 1
|
|
22461
|
+
if state != "ok":
|
|
22462
|
+
report(
|
|
22463
|
+
f"{label}: `{host}` was read as a context word. It contains "
|
|
22464
|
+
"one, which is not the same thing, and the token beside it "
|
|
22465
|
+
"here is a contract anchor. The author's only repair is to "
|
|
22466
|
+
"reword a line that was correct."
|
|
22467
|
+
)
|
|
22468
|
+
for error in errors:
|
|
22469
|
+
report(f" {error}")
|
|
22470
|
+
return 1
|
|
22471
|
+
|
|
22472
|
+
# The control. A check that passes because nothing is refused any more
|
|
22473
|
+
# is indistinguishable from this one, so the inflections the owner
|
|
22474
|
+
# decided to keep are asserted in the same repository.
|
|
22475
|
+
for host in ("commit", "commits", "committed", "committing", "hashes"):
|
|
22476
|
+
state, errors = check(
|
|
22477
|
+
"- [x] A1 implementation\n"
|
|
22478
|
+
f" - Evidence:\n"
|
|
22479
|
+
f" - M1: pass — {host} a1b2c3d4e5f6 verified.\n"
|
|
22480
|
+
)
|
|
22481
|
+
if state == "unreported":
|
|
22482
|
+
report(f"{label}: keel --check reported no state at all on `{host}`.")
|
|
22483
|
+
return 1
|
|
22484
|
+
if state != "failed":
|
|
22485
|
+
report(
|
|
22486
|
+
f"{label}: an identifier beside `{host}` was accepted. That "
|
|
22487
|
+
"word names the act the rule exists to catch, and requiring "
|
|
22488
|
+
"a boundary must not stop it counting."
|
|
22489
|
+
)
|
|
22490
|
+
return 1
|
|
22491
|
+
if not [error for error in errors if "tasks.md:" in error]:
|
|
22492
|
+
report(f"{label}: the refusal on `{host}` named no line.")
|
|
22493
|
+
return 1
|
|
22494
|
+
|
|
22495
|
+
# The Chinese words carry no boundary, because none is definable
|
|
22496
|
+
# between two word characters. Asserted here rather than reasoned
|
|
22497
|
+
# about: `\b提交\b` matches none of these.
|
|
22498
|
+
for recorded in ("已提交", "未提交", "该任务尚未提交,等待评审"):
|
|
22499
|
+
state, _ = check(
|
|
22500
|
+
"- [x] A1 implementation\n"
|
|
22501
|
+
" - Evidence:\n"
|
|
22502
|
+
f" - M1: {recorded}。\n"
|
|
22503
|
+
)
|
|
22504
|
+
if state == "unreported":
|
|
22505
|
+
report(
|
|
22506
|
+
f"{label}: keel --check reported no state at all on "
|
|
22507
|
+
f"`{recorded}`. This is not a verdict about the fixture — "
|
|
22508
|
+
"the check did not reach the point of having one."
|
|
22509
|
+
)
|
|
22510
|
+
return 1
|
|
22511
|
+
if state != "failed":
|
|
22512
|
+
report(
|
|
22513
|
+
f"{label}: recorded state written as `{recorded}` was "
|
|
22514
|
+
"accepted. A word boundary around a Chinese context word "
|
|
22515
|
+
"disables it silently, which is why it does not carry one."
|
|
22516
|
+
)
|
|
22517
|
+
return 1
|
|
22518
|
+
|
|
22519
|
+
if label not in {name for name, _ in SCENARIOS}:
|
|
22520
|
+
report(f"{label}: the scenario registry does not include it.")
|
|
22521
|
+
return 1
|
|
22522
|
+
report(f"{label} scenario passed.")
|
|
22523
|
+
return 0
|
|
22524
|
+
|
|
22525
|
+
|
|
22526
|
+
def validate_a_covers_citation_is_not_a_record_scenario() -> int:
|
|
22527
|
+
"""Issue #65 §4: the rule refused a tasks.md that cited it.
|
|
22528
|
+
|
|
22529
|
+
A `Covers` entry names a requirement that must exist in a spec. When the
|
|
22530
|
+
requirement is about the state this check forbids — dirty worktrees,
|
|
22531
|
+
recorded identifiers — citing it by name reads to the check as recording
|
|
22532
|
+
it. The repair available to the author is to rename the requirement, which
|
|
22533
|
+
is what 5.19.0 did.
|
|
22534
|
+
|
|
22535
|
+
Eleven lines across four changes in this repository's history sit in this
|
|
22536
|
+
shape, eight of them citing requirements published in `openspec/specs/`.
|
|
22537
|
+
"""
|
|
22538
|
+
label = "a-covers-citation-is-not-a-record"
|
|
22539
|
+
# Two requirement names published in this repository today. Neither is
|
|
22540
|
+
# invented for the fixture; both are cited by archived changes.
|
|
22541
|
+
cited = (
|
|
22542
|
+
"keel-core-gates / Dirty-worktree attribution is conservative"
|
|
22543
|
+
" / A path already dirty at task start is not attributed",
|
|
22544
|
+
"keel-stateless-continuity / A recorded commit hash is recognized by"
|
|
22545
|
+
" what makes it one / A decimal run is not a hash",
|
|
22546
|
+
)
|
|
22547
|
+
|
|
22548
|
+
with tempfile.TemporaryDirectory(prefix="keel-covers-citation-") as raw:
|
|
22549
|
+
check, failure = _tasks_semantics_probe(raw)
|
|
22550
|
+
if failure is not None:
|
|
22551
|
+
report(f"{label}: keel --install failed while building the fixture.")
|
|
22552
|
+
report(failure)
|
|
22553
|
+
return 1
|
|
22554
|
+
|
|
22555
|
+
state, errors = check(
|
|
22556
|
+
"- [x] A1 implementation\n"
|
|
22557
|
+
" - Covers:\n"
|
|
22558
|
+
+ "".join(f" - {entry}\n" for entry in cited)
|
|
22559
|
+
+ " - Touch:\n - src/a.js\n"
|
|
22560
|
+
)
|
|
22561
|
+
if state == "unreported":
|
|
22562
|
+
report(f"{label}: keel --check reported no state at all on a citation.")
|
|
22563
|
+
return 1
|
|
22564
|
+
if state != "ok":
|
|
22565
|
+
report(
|
|
22566
|
+
f"{label}: a Covers citation was read as a record of the state "
|
|
22567
|
+
"it names. The entry is a reference to a requirement that has "
|
|
22568
|
+
"to exist elsewhere, and the only repair open to the author is "
|
|
22569
|
+
"to rename that requirement."
|
|
22570
|
+
)
|
|
22571
|
+
for error in errors:
|
|
22572
|
+
report(f" {error}")
|
|
22573
|
+
return 1
|
|
22574
|
+
|
|
22575
|
+
# The control, and the boundary. The same wording outside the field is
|
|
22576
|
+
# a statement, and is still refused — otherwise this scenario would
|
|
22577
|
+
# pass just as well against a check that had stopped running.
|
|
22578
|
+
state, errors = check(
|
|
22579
|
+
"- [x] A1 implementation\n"
|
|
22580
|
+
" - Covers:\n"
|
|
22581
|
+
+ "".join(f" - {entry}\n" for entry in cited)
|
|
22582
|
+
+ " - Evidence:\n"
|
|
22583
|
+
" - M1: pass — the worktree was dirty when the task started.\n"
|
|
22584
|
+
)
|
|
22585
|
+
if state == "unreported":
|
|
22586
|
+
report(
|
|
22587
|
+
f"{label}: keel --check reported no state at all while reading "
|
|
22588
|
+
"the wording outside the field."
|
|
22589
|
+
)
|
|
22590
|
+
return 1
|
|
22591
|
+
if state != "failed":
|
|
22592
|
+
report(
|
|
22593
|
+
f"{label}: the same wording in an Evidence line was accepted. "
|
|
22594
|
+
"The exemption is the Covers field, not the file that contains "
|
|
22595
|
+
"one."
|
|
22596
|
+
)
|
|
22597
|
+
return 1
|
|
22598
|
+
named = [error for error in errors if "tasks.md:" in error]
|
|
22599
|
+
if not named:
|
|
22600
|
+
report(f"{label}: the refusal outside the field named no line.")
|
|
22601
|
+
return 1
|
|
22602
|
+
# The citations are the two lines under the label, and each carries
|
|
22603
|
+
# wording this rule refuses; naming either of them means the exemption
|
|
22604
|
+
# did not hold while the Evidence line was being refused.
|
|
22605
|
+
if any(f"tasks.md:{cited_line}:" in error
|
|
22606
|
+
for error in named for cited_line in (11, 12)):
|
|
22607
|
+
report(
|
|
22608
|
+
f"{label}: the refusal named a citation line. The rule fired "
|
|
22609
|
+
"correctly on the Evidence line and wrongly on the field, so "
|
|
22610
|
+
"what is wrong is the field's bound."
|
|
22611
|
+
)
|
|
22612
|
+
for error in named:
|
|
22613
|
+
report(f" {error}")
|
|
22614
|
+
return 1
|
|
22615
|
+
|
|
22616
|
+
# The bound ends where the next field begins. A `Covers` field that
|
|
22617
|
+
# ran to the end of the task would swallow the Evidence line above and
|
|
22618
|
+
# make that control vacuous, so the end of the field is asserted from
|
|
22619
|
+
# the other side too.
|
|
22620
|
+
state, _ = check(
|
|
22621
|
+
"- [x] A1 implementation\n"
|
|
22622
|
+
" - Covers:\n"
|
|
22623
|
+
f" - {cited[0]}\n"
|
|
22624
|
+
" - Verify:\n"
|
|
22625
|
+
" - M1: the change is 已提交 and needs no further work.\n"
|
|
22626
|
+
)
|
|
22627
|
+
if state == "unreported":
|
|
22628
|
+
report(
|
|
22629
|
+
f"{label}: keel --check reported no state at all while reading "
|
|
22630
|
+
"the field that follows a Covers field."
|
|
22631
|
+
)
|
|
22632
|
+
return 1
|
|
22633
|
+
if state != "failed":
|
|
22634
|
+
report(
|
|
22635
|
+
f"{label}: a Verify line following a Covers field was exempted. "
|
|
22636
|
+
"The field ends at the next field label."
|
|
22637
|
+
)
|
|
22638
|
+
return 1
|
|
22639
|
+
|
|
22640
|
+
if label not in {name for name, _ in SCENARIOS}:
|
|
22641
|
+
report(f"{label}: the scenario registry does not include it.")
|
|
22642
|
+
return 1
|
|
22643
|
+
report(f"{label} scenario passed.")
|
|
22644
|
+
return 0
|
|
22645
|
+
|
|
22646
|
+
|
|
19969
22647
|
def validate_validation_runner_scenario() -> int:
|
|
19970
22648
|
if "SCENARIOS" not in globals():
|
|
19971
22649
|
report("validation-runner: the scenario registry is missing.")
|
|
@@ -20104,6 +22782,200 @@ def validate_doctor_openspec_honesty_scenario() -> int:
|
|
|
20104
22782
|
return 0
|
|
20105
22783
|
|
|
20106
22784
|
|
|
22785
|
+
# A scenario name, as the registry spells one. Two registered names carry no
|
|
22786
|
+
# hyphen — `cli` and `uninstall` — so requiring one would leave exactly those
|
|
22787
|
+
# two unchecked, and allowing single words was measured to add no false
|
|
22788
|
+
# positive.
|
|
22789
|
+
AUTHORED_SCENARIO_NAME = re.compile(r"^[a-z0-9][a-z0-9-]*$")
|
|
22790
|
+
|
|
22791
|
+
# The two forms a scenario reference is read in. `--scenario` is the invocation
|
|
22792
|
+
# itself and needs no context. The other is keyed on the *assertion* rather than
|
|
22793
|
+
# on the token, because kebab-case is this project's spelling for gate stages,
|
|
22794
|
+
# diagnostic codes, skills, capabilities, and hook events too: reading every
|
|
22795
|
+
# such token as a scenario name flags 22 of the 55 in the archive. Keying on
|
|
22796
|
+
# "stays green" flags one, and that one is a real unregistered name.
|
|
22797
|
+
AUTHORED_SCENARIO_FLAG = re.compile(r"--scenario\s+`?([A-Za-z0-9][\w.-]*)`?")
|
|
22798
|
+
AUTHORED_SCENARIO_STAYS_GREEN = re.compile(r"stays? green", re.IGNORECASE)
|
|
22799
|
+
AUTHORED_CHECK_LINE = re.compile(r"^-\s*M\d+\b")
|
|
22800
|
+
AUTHORED_VERIFY_BLOCK = re.compile(r"^-\s*(?:Verify|Commands):\s*$")
|
|
22801
|
+
AUTHORED_CAPSULE_BLOCK = re.compile(r"^-\s*[A-Z][A-Za-z ]*:\s*$")
|
|
22802
|
+
AUTHORED_INLINE_CODE = re.compile(r"`([^`\n]+)`")
|
|
22803
|
+
|
|
22804
|
+
AUTHORED_SCENARIO_FORMS = (
|
|
22805
|
+
"A scenario reference is read in two forms: a name after `--scenario`, and "
|
|
22806
|
+
"a backticked lowercase token in a `Verify` M<n> check asserting that "
|
|
22807
|
+
"something stays green. Other backticked vocabulary in a check is not read "
|
|
22808
|
+
"as a scenario name."
|
|
22809
|
+
)
|
|
22810
|
+
|
|
22811
|
+
|
|
22812
|
+
def unregistered_scenario_references(
|
|
22813
|
+
text: str, registered: set[str]
|
|
22814
|
+
) -> list[tuple[int, str, str]]:
|
|
22815
|
+
"""Scenario names a task contract declares that the registry does not hold.
|
|
22816
|
+
|
|
22817
|
+
Returns `(line, name, form)` for each, 1-indexed, in file order. The `form`
|
|
22818
|
+
is which of the two recognized shapes matched, so a report can say why the
|
|
22819
|
+
token was read as a scenario name at all.
|
|
22820
|
+
"""
|
|
22821
|
+
problems: list[tuple[int, str, str]] = []
|
|
22822
|
+
in_verify = False
|
|
22823
|
+
for line_number, line in enumerate(text.splitlines(), start=1):
|
|
22824
|
+
stripped = line.strip()
|
|
22825
|
+
# A capsule block heading ends the previous block, so `Evidence` and
|
|
22826
|
+
# `Stop Rules` leave the Verify block rather than extending it. Evidence
|
|
22827
|
+
# is written after the run, when execution has already answered the
|
|
22828
|
+
# question this check exists to answer earlier.
|
|
22829
|
+
if AUTHORED_CAPSULE_BLOCK.match(stripped):
|
|
22830
|
+
in_verify = bool(AUTHORED_VERIFY_BLOCK.match(stripped))
|
|
22831
|
+
for match in AUTHORED_SCENARIO_FLAG.finditer(line):
|
|
22832
|
+
name = match.group(1)
|
|
22833
|
+
if name not in registered:
|
|
22834
|
+
problems.append((line_number, name, "--scenario"))
|
|
22835
|
+
if not in_verify or not AUTHORED_CHECK_LINE.match(stripped):
|
|
22836
|
+
continue
|
|
22837
|
+
if not AUTHORED_SCENARIO_STAYS_GREEN.search(line):
|
|
22838
|
+
continue
|
|
22839
|
+
for token in AUTHORED_INLINE_CODE.findall(line):
|
|
22840
|
+
if not AUTHORED_SCENARIO_NAME.match(token):
|
|
22841
|
+
continue
|
|
22842
|
+
if token not in registered:
|
|
22843
|
+
problems.append((line_number, token, "stays green"))
|
|
22844
|
+
return problems
|
|
22845
|
+
|
|
22846
|
+
|
|
22847
|
+
def active_change_tasks(repo: Path) -> list[Path]:
|
|
22848
|
+
"""Every active change's tasks.md. Archived changes are deliberately absent.
|
|
22849
|
+
|
|
22850
|
+
An archived task records what was true when it ran; a scenario renamed
|
|
22851
|
+
afterwards does not make that record wrong, and turning history red for it
|
|
22852
|
+
tells a reader nothing they can act on.
|
|
22853
|
+
"""
|
|
22854
|
+
changes = repo / "openspec" / "changes"
|
|
22855
|
+
archive = changes / "archive"
|
|
22856
|
+
# The walk finds archived changes and the filter is what drops them, rather
|
|
22857
|
+
# than a glob depth that happens not to reach them. A depth that excludes by
|
|
22858
|
+
# accident passes every test aimed at the exclusion, including one aimed at
|
|
22859
|
+
# it on purpose.
|
|
22860
|
+
return sorted(
|
|
22861
|
+
path for path in changes.rglob("tasks.md") if archive not in path.parents
|
|
22862
|
+
)
|
|
22863
|
+
|
|
22864
|
+
|
|
22865
|
+
def validate_authored_scenario_names_scenario() -> int:
|
|
22866
|
+
"""Issue #51: a check whose scenario name is not registered cannot run.
|
|
22867
|
+
|
|
22868
|
+
Three of the four authored contracts #51 records named something that did
|
|
22869
|
+
not exist, two of them scenario names, both written as an `M5 (regression)`
|
|
22870
|
+
check asserting that named scenarios stay green, and both found only when
|
|
22871
|
+
the check was executed. A fifth, unrecorded there, is
|
|
22872
|
+
`2026-08-01-the-name-is-not-the-thing/tasks.md:132`, which declares
|
|
22873
|
+
`tasks-template-validates` — a name `git log -S` finds at no commit — and
|
|
22874
|
+
records that check as passing.
|
|
22875
|
+
|
|
22876
|
+
The assertions live on synthetic content with a known name set, because the
|
|
22877
|
+
live scan reads whatever the working tree holds and is normally empty: a
|
|
22878
|
+
change is archived when it closes. The temp-repository half is the positive
|
|
22879
|
+
control for the whole path, so a run that passes because the file selection
|
|
22880
|
+
silently found nothing is distinguishable from one that verified something.
|
|
22881
|
+
"""
|
|
22882
|
+
registered = {name for name, _ in SCENARIOS}
|
|
22883
|
+
known = {"core-gates", "cli"}
|
|
22884
|
+
authored = (
|
|
22885
|
+
"# Tasks\n"
|
|
22886
|
+
"\n"
|
|
22887
|
+
"- [ ] 1.1 Do the thing\n"
|
|
22888
|
+
" - Touch:\n"
|
|
22889
|
+
" - src/thing.js\n"
|
|
22890
|
+
" - Verify:\n"
|
|
22891
|
+
" - Strategy: vertical-tdd\n"
|
|
22892
|
+
" - M1: `node scripts/run_python.js scripts/validate_plugin.py "
|
|
22893
|
+
"--scenario gate-diagnostics` passes\n"
|
|
22894
|
+
" - M2 (regression): `core-gates` and `target-surface-doctor` stay "
|
|
22895
|
+
"green\n"
|
|
22896
|
+
" - M3 (regression): `task-start`, `keel-review-checklist`, "
|
|
22897
|
+
"`contract-drift`, `subagent-stop`, and `keel-task-capsule` are "
|
|
22898
|
+
"unchanged\n"
|
|
22899
|
+
" - M4: `cli` and "
|
|
22900
|
+
"`node scripts/run_python.js scripts/validate_plugin.py "
|
|
22901
|
+
"--scenario core-gates` stay green\n"
|
|
22902
|
+
" - Evidence:\n"
|
|
22903
|
+
" - M2: `target-surface-doctor` stays green\n"
|
|
22904
|
+
)
|
|
22905
|
+
found = unregistered_scenario_references(authored, known)
|
|
22906
|
+
expected = [
|
|
22907
|
+
(8, "gate-diagnostics", "--scenario"),
|
|
22908
|
+
(9, "target-surface-doctor", "stays green"),
|
|
22909
|
+
]
|
|
22910
|
+
if found != expected:
|
|
22911
|
+
report(
|
|
22912
|
+
"authored-scenario-names-are-registered: the recognized forms did "
|
|
22913
|
+
f"not report what they must. expected {expected}, found {found}."
|
|
22914
|
+
)
|
|
22915
|
+
report(AUTHORED_SCENARIO_FORMS)
|
|
22916
|
+
return 1
|
|
22917
|
+
|
|
22918
|
+
# Which files are read is a separate question from what is read out of
|
|
22919
|
+
# them, and it is the half that can silently resolve to nothing. The
|
|
22920
|
+
# planted reference is the positive control for the whole path: an
|
|
22921
|
+
# extractor that stopped reporting, or a selection that stopped finding,
|
|
22922
|
+
# both fail here rather than passing on an empty scan.
|
|
22923
|
+
with tempfile.TemporaryDirectory(prefix="keel-authored-names-") as raw:
|
|
22924
|
+
repo = Path(raw)
|
|
22925
|
+
write_text(repo / "openspec/changes/live/tasks.md", authored)
|
|
22926
|
+
write_text(repo / "openspec/changes/archive/old/tasks.md", authored)
|
|
22927
|
+
selected = active_change_tasks(repo)
|
|
22928
|
+
if selected != [repo / "openspec/changes/live/tasks.md"]:
|
|
22929
|
+
report(
|
|
22930
|
+
"authored-scenario-names-are-registered: the active change "
|
|
22931
|
+
"selection is wrong. An archived tasks.md must not be scanned "
|
|
22932
|
+
"and an active one must be. "
|
|
22933
|
+
f"selected {[str(path) for path in selected]}."
|
|
22934
|
+
)
|
|
22935
|
+
return 1
|
|
22936
|
+
control = [
|
|
22937
|
+
(path, unregistered_scenario_references(
|
|
22938
|
+
path.read_text(encoding="utf-8"), known
|
|
22939
|
+
))
|
|
22940
|
+
for path in selected
|
|
22941
|
+
]
|
|
22942
|
+
if [problems for _, problems in control] != [expected]:
|
|
22943
|
+
report(
|
|
22944
|
+
"authored-scenario-names-are-registered: a planted reference "
|
|
22945
|
+
"in an active change was not reported end to end, so a clean "
|
|
22946
|
+
"run over the real tree would prove nothing. "
|
|
22947
|
+
f"found {control}."
|
|
22948
|
+
)
|
|
22949
|
+
return 1
|
|
22950
|
+
|
|
22951
|
+
for path in active_change_tasks(ROOT):
|
|
22952
|
+
problems = unregistered_scenario_references(
|
|
22953
|
+
path.read_text(encoding="utf-8"), registered
|
|
22954
|
+
)
|
|
22955
|
+
if not problems:
|
|
22956
|
+
continue
|
|
22957
|
+
relative = path.relative_to(ROOT).as_posix()
|
|
22958
|
+
for line_number, name, form in problems:
|
|
22959
|
+
report(
|
|
22960
|
+
f"authored-scenario-names-are-registered: {relative}:"
|
|
22961
|
+
f"{line_number} names `{name}` after {form}, and no scenario by "
|
|
22962
|
+
"that name is registered, so the check that names it cannot run."
|
|
22963
|
+
)
|
|
22964
|
+
report(AUTHORED_SCENARIO_FORMS)
|
|
22965
|
+
return 1
|
|
22966
|
+
|
|
22967
|
+
if "authored-scenario-names-are-registered" not in {
|
|
22968
|
+
name for name, _ in SCENARIOS
|
|
22969
|
+
}:
|
|
22970
|
+
report(
|
|
22971
|
+
"authored-scenario-names-are-registered: the scenario registry "
|
|
22972
|
+
"does not include it."
|
|
22973
|
+
)
|
|
22974
|
+
return 1
|
|
22975
|
+
report("authored-scenario-names-are-registered scenario passed.")
|
|
22976
|
+
return 0
|
|
22977
|
+
|
|
22978
|
+
|
|
20107
22979
|
SCENARIOS: tuple = (
|
|
20108
22980
|
("stateless-continuity", validate_stateless_continuity_scenario),
|
|
20109
22981
|
("core-gates", validate_core_gates_scenario),
|
|
@@ -20113,6 +22985,10 @@ SCENARIOS: tuple = (
|
|
|
20113
22985
|
("target-surface", validate_target_surface_scenario),
|
|
20114
22986
|
("thin-native-install", validate_thin_native_install_scenario),
|
|
20115
22987
|
("expectation-slice-gates", validate_expectation_slice_gates_scenario),
|
|
22988
|
+
(
|
|
22989
|
+
"unparsed-covers-critical-statement",
|
|
22990
|
+
validate_unparsed_covers_critical_statement_scenario,
|
|
22991
|
+
),
|
|
20116
22992
|
("expectation-completion-gates", validate_expectation_completion_gates_scenario),
|
|
20117
22993
|
("authoring-continuity", validate_authoring_continuity_scenario),
|
|
20118
22994
|
("domain-lenses", validate_domain_lenses_scenario),
|
|
@@ -20121,8 +22997,16 @@ SCENARIOS: tuple = (
|
|
|
20121
22997
|
("sync-surface-overlay", validate_sync_surface_overlay_scenario),
|
|
20122
22998
|
("openspec-surface-overlay", validate_openspec_surface_overlay_scenario),
|
|
20123
22999
|
("uninstall", validate_uninstall_scenario),
|
|
23000
|
+
(
|
|
23001
|
+
"uninstall-removes-the-overlay",
|
|
23002
|
+
validate_uninstall_removes_the_overlay_scenario,
|
|
23003
|
+
),
|
|
20124
23004
|
("cli", validate_cli_scenario),
|
|
20125
23005
|
("doctor-openspec-honesty", validate_doctor_openspec_honesty_scenario),
|
|
23006
|
+
(
|
|
23007
|
+
"authored-scenario-names-are-registered",
|
|
23008
|
+
validate_authored_scenario_names_scenario,
|
|
23009
|
+
),
|
|
20126
23010
|
("update-pack-install", validate_update_pack_install_scenario),
|
|
20127
23011
|
("update-default-registry", validate_update_default_registry_scenario),
|
|
20128
23012
|
("verification-layering-docs", validate_verification_layering_docs_scenario),
|
|
@@ -20130,6 +23014,10 @@ SCENARIOS: tuple = (
|
|
|
20130
23014
|
"standing-authorization-declaration",
|
|
20131
23015
|
validate_standing_authorization_declaration_scenario,
|
|
20132
23016
|
),
|
|
23017
|
+
(
|
|
23018
|
+
"standing-authorization-sync-confusion",
|
|
23019
|
+
validate_standing_authorization_sync_confusion_scenario,
|
|
23020
|
+
),
|
|
20133
23021
|
(
|
|
20134
23022
|
"standing-authorization-inheritance",
|
|
20135
23023
|
validate_standing_authorization_inheritance_scenario,
|
|
@@ -20145,6 +23033,10 @@ SCENARIOS: tuple = (
|
|
|
20145
23033
|
("precedent-never-weakens", validate_precedent_never_weakens_scenario),
|
|
20146
23034
|
("precedent-rules", validate_precedent_rules_scenario),
|
|
20147
23035
|
("triage-declaration", validate_triage_declaration_scenario),
|
|
23036
|
+
(
|
|
23037
|
+
"triage-admits-from-the-repository",
|
|
23038
|
+
validate_triage_admits_from_the_repository_scenario,
|
|
23039
|
+
),
|
|
20148
23040
|
("delegation-declaration", validate_delegation_declaration_scenario),
|
|
20149
23041
|
("delegation-inheritance", validate_delegation_inheritance_scenario),
|
|
20150
23042
|
("native-capability-scope", validate_native_capability_scope_scenario),
|
|
@@ -20175,6 +23067,10 @@ SCENARIOS: tuple = (
|
|
|
20175
23067
|
),
|
|
20176
23068
|
("inline-code-is-concrete", validate_inline_code_is_concrete_scenario),
|
|
20177
23069
|
("covers-separator-collision", validate_covers_separator_collision_scenario),
|
|
23070
|
+
(
|
|
23071
|
+
"unresolved-covers-names-what-failed",
|
|
23072
|
+
validate_unresolved_covers_names_what_failed_scenario,
|
|
23073
|
+
),
|
|
20178
23074
|
(
|
|
20179
23075
|
"unresolved-authority-names-field",
|
|
20180
23076
|
validate_unresolved_authority_names_field_scenario,
|
|
@@ -20251,12 +23147,20 @@ SCENARIOS: tuple = (
|
|
|
20251
23147
|
("tracker-durable-owner", validate_tracker_durable_owner_scenario),
|
|
20252
23148
|
("findings-resolved-here", validate_findings_resolved_here_scenario),
|
|
20253
23149
|
("guard-stale-manifest", validate_guard_stale_manifest_scenario),
|
|
23150
|
+
(
|
|
23151
|
+
"guard-status-stale-manifest",
|
|
23152
|
+
validate_guard_status_stale_manifest_scenario,
|
|
23153
|
+
),
|
|
20254
23154
|
("assertion-shape-count", validate_assertion_shape_count_scenario),
|
|
20255
23155
|
("guard-manifest-ignored", validate_guard_manifest_ignored_scenario),
|
|
20256
23156
|
(
|
|
20257
23157
|
"guard-status-is-not-enforcement",
|
|
20258
23158
|
validate_guard_status_is_not_enforcement_scenario,
|
|
20259
23159
|
),
|
|
23160
|
+
(
|
|
23161
|
+
"guard-warnings-are-concise",
|
|
23162
|
+
validate_guard_warnings_are_concise_scenario,
|
|
23163
|
+
),
|
|
20260
23164
|
("source-repo-cli-resolution", validate_source_repo_cli_resolution_scenario),
|
|
20261
23165
|
("task-contract-core", validate_task_contract_core_scenario),
|
|
20262
23166
|
("task-capsule", validate_task_capsule_scenario),
|
|
@@ -20332,7 +23236,19 @@ SCENARIOS: tuple = (
|
|
|
20332
23236
|
"decimal-runs-are-not-hash-shaped",
|
|
20333
23237
|
validate_decimal_runs_are_not_hash_shaped_scenario,
|
|
20334
23238
|
),
|
|
23239
|
+
("a-context-word-is-a-word", validate_a_context_word_is_a_word_scenario),
|
|
23240
|
+
(
|
|
23241
|
+
"a-covers-citation-is-not-a-record",
|
|
23242
|
+
validate_a_covers_citation_is_not_a_record_scenario,
|
|
23243
|
+
),
|
|
20335
23244
|
("validation-runner", validate_validation_runner_scenario),
|
|
23245
|
+
("section-boundary", validate_section_boundary_scenario),
|
|
23246
|
+
("review-entry-extent", validate_review_entry_extent_scenario),
|
|
23247
|
+
("reauthorizations-shape", validate_reauthorizations_shape_scenario),
|
|
23248
|
+
(
|
|
23249
|
+
"change-verify-deferred-evidence",
|
|
23250
|
+
validate_change_verify_deferred_evidence_scenario,
|
|
23251
|
+
),
|
|
20336
23252
|
)
|
|
20337
23253
|
|
|
20338
23254
|
|