@christang/keel 5.16.0 → 5.39.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +28 -9
- package/assets/bootstrap/AGENTS.md +1 -1
- package/assets/openspec/schemas/keel-spec-driven/templates/tasks.md +4 -1
- package/bin/keel.js +127 -22
- package/package.json +1 -1
- package/plugins/keel/.claude-plugin/plugin.json +1 -1
- package/plugins/keel/.codex-plugin/plugin.json +1 -1
- package/plugins/keel/skills/keel-align-expectations/SKILL.md +4 -18
- package/plugins/keel/skills/keel-review-checklist/SKILL.md +1 -1
- package/scripts/install_to_repo.py +56 -3
- package/scripts/validate_plugin.py +3782 -228
- package/src/core/config.js +192 -26
- package/src/core/context.js +9 -0
- package/src/core/gates.js +305 -54
- package/src/core/guard.js +77 -16
- package/src/core/task-contract.js +103 -7
|
@@ -37,8 +37,8 @@ REQUIRED_SCRIPTS = [
|
|
|
37
37
|
"scripts/validate_plugin.py",
|
|
38
38
|
]
|
|
39
39
|
|
|
40
|
-
PACKAGE_VERSION = "5.
|
|
41
|
-
PROTOCOL_VERSION = "5.
|
|
40
|
+
PACKAGE_VERSION = "5.39.0"
|
|
41
|
+
PROTOCOL_VERSION = "5.39.0"
|
|
42
42
|
LEGACY_MANAGED_START = "<!-- keel:start version=2.1 -->"
|
|
43
43
|
OPENSPEC_SCHEMA_NAME = "keel-spec-driven"
|
|
44
44
|
# Mirrors KEEL_PACKAGE_NAME in scripts/install_to_repo.py, one of the two
|
|
@@ -1611,6 +1611,118 @@ def validate_expectation_slice_gates_scenario() -> int:
|
|
|
1611
1611
|
return 0
|
|
1612
1612
|
|
|
1613
1613
|
|
|
1614
|
+
def validate_unparsed_covers_critical_statement_scenario() -> int:
|
|
1615
|
+
"""Issue #49 item 1: a critical-statement Covers reference must be told apart
|
|
1616
|
+
as missing (the identifier never appears in design.md) from unparsed (the
|
|
1617
|
+
identifier appears but not in the exact resolvable shape), instead of both
|
|
1618
|
+
collapsing into the same "Missing" message.
|
|
1619
|
+
"""
|
|
1620
|
+
task = task_capsule_compact_fixture().replace(
|
|
1621
|
+
" - E1: Public behavior passes.\n",
|
|
1622
|
+
" - D2\n",
|
|
1623
|
+
)
|
|
1624
|
+
|
|
1625
|
+
def covers_message(repo: Path) -> tuple[int, str]:
|
|
1626
|
+
write_text(repo / "openspec/changes/demo/tasks.md", task)
|
|
1627
|
+
result = run_keel(
|
|
1628
|
+
repo, "gate", "task-start", "--change", "demo", "--task", "1.1", "--json"
|
|
1629
|
+
)
|
|
1630
|
+
problems = json.loads(result.stdout).get("problems", [])
|
|
1631
|
+
message = next(
|
|
1632
|
+
(
|
|
1633
|
+
item.get("message", "")
|
|
1634
|
+
for item in problems
|
|
1635
|
+
if item.get("code") == "unresolved-covers"
|
|
1636
|
+
),
|
|
1637
|
+
"",
|
|
1638
|
+
)
|
|
1639
|
+
return result.returncode, message
|
|
1640
|
+
|
|
1641
|
+
with tempfile.TemporaryDirectory(prefix="keel-unparsed-covers-missing-") as raw:
|
|
1642
|
+
repo = Path(raw)
|
|
1643
|
+
write_text(
|
|
1644
|
+
repo / "openspec/changes/demo/design.md",
|
|
1645
|
+
"## Decisions\n\nD1 — Unrelated decision. Basis: fixture authority.\n",
|
|
1646
|
+
)
|
|
1647
|
+
returncode, message = covers_message(repo)
|
|
1648
|
+
if returncode == 0 or not message.startswith(
|
|
1649
|
+
"Missing Covers critical statement: D2."
|
|
1650
|
+
):
|
|
1651
|
+
report(
|
|
1652
|
+
"unparsed-covers-critical-statement: an identifier absent from "
|
|
1653
|
+
f"design.md must still report Missing, got: {message!r}"
|
|
1654
|
+
)
|
|
1655
|
+
return 1
|
|
1656
|
+
|
|
1657
|
+
with tempfile.TemporaryDirectory(prefix="keel-unparsed-covers-unparsed-") as raw:
|
|
1658
|
+
repo = Path(raw)
|
|
1659
|
+
write_text(
|
|
1660
|
+
repo / "openspec/changes/demo/design.md",
|
|
1661
|
+
"## Decisions\n\n"
|
|
1662
|
+
"- **D2** — Keep one shared parser. Basis: fixture authority.\n",
|
|
1663
|
+
)
|
|
1664
|
+
returncode, message = covers_message(repo)
|
|
1665
|
+
if returncode == 0:
|
|
1666
|
+
report(
|
|
1667
|
+
"unparsed-covers-critical-statement: a present-but-mis-shaped D2 "
|
|
1668
|
+
"must still fail task-start."
|
|
1669
|
+
)
|
|
1670
|
+
return 1
|
|
1671
|
+
if not message.startswith("Unparsed Covers critical statement: D2."):
|
|
1672
|
+
report(
|
|
1673
|
+
"unparsed-covers-critical-statement: a present-but-mis-shaped D2 "
|
|
1674
|
+
f"must report Unparsed, got: {message!r}"
|
|
1675
|
+
)
|
|
1676
|
+
return 1
|
|
1677
|
+
if "D2 — one-line statement" not in message:
|
|
1678
|
+
report(
|
|
1679
|
+
"unparsed-covers-critical-statement: the Unparsed message must "
|
|
1680
|
+
f"name the required shape, got: {message!r}"
|
|
1681
|
+
)
|
|
1682
|
+
return 1
|
|
1683
|
+
|
|
1684
|
+
with tempfile.TemporaryDirectory(prefix="keel-unparsed-covers-resolved-") as raw:
|
|
1685
|
+
repo = Path(raw)
|
|
1686
|
+
write_text(
|
|
1687
|
+
repo / "openspec/changes/demo/design.md",
|
|
1688
|
+
"## Decisions\n\nD2 — Keep one shared parser. Basis: fixture authority.\n",
|
|
1689
|
+
)
|
|
1690
|
+
write_text(repo / "openspec/changes/demo/tasks.md", task)
|
|
1691
|
+
result = run_keel(
|
|
1692
|
+
repo, "gate", "task-start", "--change", "demo", "--task", "1.1", "--json"
|
|
1693
|
+
)
|
|
1694
|
+
if result.returncode != 0:
|
|
1695
|
+
report(
|
|
1696
|
+
"unparsed-covers-critical-statement: a correctly-shaped D2 must "
|
|
1697
|
+
f"still resolve. {(result.stderr or result.stdout).strip()}"
|
|
1698
|
+
)
|
|
1699
|
+
return 1
|
|
1700
|
+
authority = (
|
|
1701
|
+
json.loads(result.stdout)
|
|
1702
|
+
.get("contract", {})
|
|
1703
|
+
.get("capsule", {})
|
|
1704
|
+
.get("authority", [])
|
|
1705
|
+
)
|
|
1706
|
+
if not any(
|
|
1707
|
+
item.get("reference") == "D2" and item.get("kind") == "critical-statement"
|
|
1708
|
+
for item in authority
|
|
1709
|
+
):
|
|
1710
|
+
report(
|
|
1711
|
+
"unparsed-covers-critical-statement: correctly-shaped D2 did not "
|
|
1712
|
+
"resolve as critical-statement authority."
|
|
1713
|
+
)
|
|
1714
|
+
return 1
|
|
1715
|
+
|
|
1716
|
+
if "unparsed-covers-critical-statement" not in {name for name, _ in SCENARIOS}:
|
|
1717
|
+
report(
|
|
1718
|
+
"unparsed-covers-critical-statement: the scenario registry does not "
|
|
1719
|
+
"include it."
|
|
1720
|
+
)
|
|
1721
|
+
return 1
|
|
1722
|
+
report("unparsed-covers-critical-statement scenario passed.")
|
|
1723
|
+
return 0
|
|
1724
|
+
|
|
1725
|
+
|
|
1614
1726
|
def validate_expectation_completion_gates_scenario() -> int:
|
|
1615
1727
|
protocol_snippets = [
|
|
1616
1728
|
"Completion Gate",
|
|
@@ -2147,6 +2259,256 @@ def assert_target_overlays(
|
|
|
2147
2259
|
return None
|
|
2148
2260
|
|
|
2149
2261
|
|
|
2262
|
+
OVERLAY_ACTION_SKILLS = {
|
|
2263
|
+
"propose": "openspec-propose",
|
|
2264
|
+
"apply": "openspec-apply-change",
|
|
2265
|
+
"archive": "openspec-archive-change",
|
|
2266
|
+
"sync": "openspec-sync-specs",
|
|
2267
|
+
}
|
|
2268
|
+
|
|
2269
|
+
|
|
2270
|
+
def expected_overlay_surfaces(
|
|
2271
|
+
repo: Path,
|
|
2272
|
+
target: str,
|
|
2273
|
+
codex_home: Path | None = None,
|
|
2274
|
+
) -> list[Path]:
|
|
2275
|
+
"""Every surface `openspecOverlaySurfacesForTarget` projects the overlay onto.
|
|
2276
|
+
|
|
2277
|
+
Written out rather than discovered, so that a scan of the tree can be
|
|
2278
|
+
compared against it. A scan alone answers "the files that carry the
|
|
2279
|
+
marker", which is the same answer whether the CLI covered eight surfaces
|
|
2280
|
+
or none — and "none of them still carry it" is exactly what a broken
|
|
2281
|
+
surface list also reports.
|
|
2282
|
+
"""
|
|
2283
|
+
surfaces: list[Path] = []
|
|
2284
|
+
for action, skill in OVERLAY_ACTION_SKILLS.items():
|
|
2285
|
+
if action == "propose" and target == "opencode":
|
|
2286
|
+
continue
|
|
2287
|
+
if target == "claude":
|
|
2288
|
+
surfaces.append(repo / f".claude/skills/{skill}/SKILL.md")
|
|
2289
|
+
surfaces.append(repo / f".claude/commands/opsx/{action}.md")
|
|
2290
|
+
elif target == "codex":
|
|
2291
|
+
assert codex_home is not None
|
|
2292
|
+
surfaces.append(repo / f".codex/skills/{skill}/SKILL.md")
|
|
2293
|
+
surfaces.append(codex_home / f"prompts/opsx-{action}.md")
|
|
2294
|
+
else:
|
|
2295
|
+
surfaces.append(repo / f".opencode/skills/{skill}/SKILL.md")
|
|
2296
|
+
surfaces.append(repo / f".opencode/commands/opsx-{action}.md")
|
|
2297
|
+
return surfaces
|
|
2298
|
+
|
|
2299
|
+
|
|
2300
|
+
def files_carrying_overlay(*roots: Path) -> set[Path]:
|
|
2301
|
+
found: set[Path] = set()
|
|
2302
|
+
for root in roots:
|
|
2303
|
+
if not root.is_dir():
|
|
2304
|
+
continue
|
|
2305
|
+
for path in root.rglob("*.md"):
|
|
2306
|
+
if not path.is_file():
|
|
2307
|
+
continue
|
|
2308
|
+
text = path.read_text(encoding="utf-8", errors="replace")
|
|
2309
|
+
if "keel:openspec-surface-overlay" in text:
|
|
2310
|
+
found.add(path)
|
|
2311
|
+
return found
|
|
2312
|
+
|
|
2313
|
+
|
|
2314
|
+
def validate_uninstall_removes_the_overlay_scenario() -> int:
|
|
2315
|
+
"""Uninstalling left Keel's instructions in files OpenSpec owns.
|
|
2316
|
+
|
|
2317
|
+
The overlay block is written into the `opsx` command surfaces and the
|
|
2318
|
+
`openspec-*` skills, and every line of it is about Keel: the gate to run,
|
|
2319
|
+
the checklist to run, how to invoke OpenSpec, where standing authorization
|
|
2320
|
+
is declared. `keel --uninstall` removed the `keel:start` managed block from
|
|
2321
|
+
AGENTS.md and CLAUDE.md and left all of that behind.
|
|
2322
|
+
|
|
2323
|
+
Measured 2026-08-03 at 5.22.0: Claude 8 of 8 surfaces, Codex 8 of 8 (four
|
|
2324
|
+
of them under `CODEX_HOME/prompts/`), OpenCode 6 of 6 still carried the
|
|
2325
|
+
marker after uninstall. Filed as
|
|
2326
|
+
https://github.com/TanglmChris/keel/issues/50.
|
|
2327
|
+
"""
|
|
2328
|
+
label = "uninstall-removes-the-overlay"
|
|
2329
|
+
with tempfile.TemporaryDirectory(prefix="keel-uninstall-overlay-") as raw:
|
|
2330
|
+
tmp = Path(raw)
|
|
2331
|
+
|
|
2332
|
+
for target in ("claude", "codex", "opencode"):
|
|
2333
|
+
repo = tmp / target
|
|
2334
|
+
repo.mkdir()
|
|
2335
|
+
env = None
|
|
2336
|
+
codex_home = None
|
|
2337
|
+
roots = [repo]
|
|
2338
|
+
if target == "codex":
|
|
2339
|
+
codex_home = tmp / "codex-home"
|
|
2340
|
+
env = os.environ.copy()
|
|
2341
|
+
env["CODEX_HOME"] = str(codex_home)
|
|
2342
|
+
roots.append(codex_home)
|
|
2343
|
+
|
|
2344
|
+
init = run_keel(repo, "--init", "--target", target, env=env)
|
|
2345
|
+
if init.returncode != 0:
|
|
2346
|
+
report(f"{label} `keel --init --target {target}` failed.")
|
|
2347
|
+
report((init.stderr or init.stdout).strip())
|
|
2348
|
+
return 1
|
|
2349
|
+
|
|
2350
|
+
# M1 positive control — the surfaces the CLI writes to are the ones
|
|
2351
|
+
# this scenario is about to check, asserted before the act. Without
|
|
2352
|
+
# it, a run whose surface list resolved to nothing reports the same
|
|
2353
|
+
# clean tree as a run that removed everything.
|
|
2354
|
+
expected = set(expected_overlay_surfaces(repo, target, codex_home))
|
|
2355
|
+
carrying = files_carrying_overlay(*roots)
|
|
2356
|
+
if carrying != expected:
|
|
2357
|
+
report(
|
|
2358
|
+
f"{label} M1 the {target} surfaces carrying the overlay "
|
|
2359
|
+
"after init are not the ones this scenario expects, so "
|
|
2360
|
+
"nothing after this measures removal."
|
|
2361
|
+
)
|
|
2362
|
+
report(f" only installed: {sorted(str(p) for p in carrying - expected)}")
|
|
2363
|
+
report(f" only expected: {sorted(str(p) for p in expected - carrying)}")
|
|
2364
|
+
return 1
|
|
2365
|
+
|
|
2366
|
+
# M2 — what the file held before Keel's block, kept so the whole
|
|
2367
|
+
# file can be compared against it afterwards. Trailing newlines are
|
|
2368
|
+
# normalized to one because the separator the install side inserted
|
|
2369
|
+
# before the block goes with the block.
|
|
2370
|
+
bodies = {}
|
|
2371
|
+
for surface in expected:
|
|
2372
|
+
text = surface.read_text(encoding="utf-8")
|
|
2373
|
+
head = text[: text.index("<!-- keel:openspec-surface-overlay")]
|
|
2374
|
+
bodies[surface] = head.rstrip("\n") + "\n"
|
|
2375
|
+
|
|
2376
|
+
uninstall = run_keel(repo, "--uninstall", "--target", target, env=env)
|
|
2377
|
+
if uninstall.returncode != 0:
|
|
2378
|
+
report(f"{label} M1 `keel --uninstall --target {target}` failed.")
|
|
2379
|
+
report((uninstall.stderr or uninstall.stdout).strip())
|
|
2380
|
+
return 1
|
|
2381
|
+
|
|
2382
|
+
# M1 — nothing Keel wrote is left in a file OpenSpec owns.
|
|
2383
|
+
left = files_carrying_overlay(*roots)
|
|
2384
|
+
if left:
|
|
2385
|
+
report(
|
|
2386
|
+
f"{label} M1 uninstalling {target} left the Keel overlay "
|
|
2387
|
+
f"in {len(left)} of {len(expected)} surfaces, so the "
|
|
2388
|
+
"uninstalled repository still instructs an agent to run "
|
|
2389
|
+
"commands that were just removed."
|
|
2390
|
+
)
|
|
2391
|
+
for path in sorted(str(p) for p in left):
|
|
2392
|
+
report(f" {path}")
|
|
2393
|
+
return 1
|
|
2394
|
+
|
|
2395
|
+
# M4 — the run says how many surfaces it cleaned. An uninstall that
|
|
2396
|
+
# silently does nothing and an uninstall that cleaned everything
|
|
2397
|
+
# produce the same tree on a repository that was never installed.
|
|
2398
|
+
if f"removed={len(expected)}" not in uninstall.stdout:
|
|
2399
|
+
report(
|
|
2400
|
+
f"{label} M4 uninstalling {target} did not report the "
|
|
2401
|
+
f"{len(expected)} overlays it removed."
|
|
2402
|
+
)
|
|
2403
|
+
report((uninstall.stderr or uninstall.stdout).strip())
|
|
2404
|
+
return 1
|
|
2405
|
+
|
|
2406
|
+
# M2 — the block, and only the block. The file is OpenSpec's, so a
|
|
2407
|
+
# removal that took the marker and anything either side of it is a
|
|
2408
|
+
# worse defect than the one being fixed, and "the marker is gone"
|
|
2409
|
+
# cannot tell the two apart.
|
|
2410
|
+
for surface, body in bodies.items():
|
|
2411
|
+
if not surface.is_file():
|
|
2412
|
+
report(
|
|
2413
|
+
f"{label} M2 uninstalling {target} deleted {surface}, "
|
|
2414
|
+
"which is OpenSpec's file. Removing Keel's block is "
|
|
2415
|
+
"the whole obligation."
|
|
2416
|
+
)
|
|
2417
|
+
return 1
|
|
2418
|
+
after = surface.read_text(encoding="utf-8")
|
|
2419
|
+
if after != body:
|
|
2420
|
+
report(
|
|
2421
|
+
f"{label} M2 uninstalling {target} did not leave "
|
|
2422
|
+
f"{surface} at the bytes that preceded the overlay."
|
|
2423
|
+
)
|
|
2424
|
+
report(f" expected {len(body)} bytes ending {body[-40:]!r}")
|
|
2425
|
+
report(f" found {len(after)} bytes ending {after[-40:]!r}")
|
|
2426
|
+
return 1
|
|
2427
|
+
|
|
2428
|
+
# M3 — a dry run plans the writes it would make. `--check` already had
|
|
2429
|
+
# to learn that a Node-side step the installer's plan cannot see makes
|
|
2430
|
+
# a dry run under-report a run that writes; the uninstall side has the
|
|
2431
|
+
# same shape.
|
|
2432
|
+
dry = tmp / "dry-run"
|
|
2433
|
+
dry.mkdir()
|
|
2434
|
+
dry_init = run_keel(dry, "--init", "--target", "claude")
|
|
2435
|
+
if dry_init.returncode != 0:
|
|
2436
|
+
report(f"{label} M3 dry-run fixture init failed.")
|
|
2437
|
+
report((dry_init.stderr or dry_init.stdout).strip())
|
|
2438
|
+
return 1
|
|
2439
|
+
dry_expected = set(expected_overlay_surfaces(dry, "claude"))
|
|
2440
|
+
dry_run = run_keel(dry, "--uninstall", "--target", "claude", "--dry-run")
|
|
2441
|
+
if dry_run.returncode != 0:
|
|
2442
|
+
report(f"{label} M3 `keel --uninstall --dry-run` failed.")
|
|
2443
|
+
report((dry_run.stderr or dry_run.stdout).strip())
|
|
2444
|
+
return 1
|
|
2445
|
+
unplanned = [
|
|
2446
|
+
str(surface)
|
|
2447
|
+
for surface in sorted(dry_expected)
|
|
2448
|
+
if str(surface) not in dry_run.stdout
|
|
2449
|
+
]
|
|
2450
|
+
if unplanned:
|
|
2451
|
+
report(
|
|
2452
|
+
f"{label} M3 the dry run did not name {len(unplanned)} of "
|
|
2453
|
+
f"{len(dry_expected)} surfaces it would clean, so the plan "
|
|
2454
|
+
"under-reports a run that writes."
|
|
2455
|
+
)
|
|
2456
|
+
for surface in unplanned:
|
|
2457
|
+
report(f" {surface}")
|
|
2458
|
+
return 1
|
|
2459
|
+
if files_carrying_overlay(dry) != dry_expected:
|
|
2460
|
+
report(
|
|
2461
|
+
f"{label} M3 the dry run removed overlays. A dry run reports "
|
|
2462
|
+
"what it would do and writes nothing."
|
|
2463
|
+
)
|
|
2464
|
+
return 1
|
|
2465
|
+
|
|
2466
|
+
# M4 — uninstalling again, and uninstalling a repository that never
|
|
2467
|
+
# received the overlay, both succeed. Uninstall is something a user
|
|
2468
|
+
# reaches for when something is already wrong; failing on the second
|
|
2469
|
+
# attempt is the worst moment to fail.
|
|
2470
|
+
again = run_keel(tmp / "claude", "--uninstall", "--target", "claude")
|
|
2471
|
+
if again.returncode != 0:
|
|
2472
|
+
report(
|
|
2473
|
+
f"{label} M4 a second uninstall failed. Uninstall is reached "
|
|
2474
|
+
"when something is already wrong; failing on the second "
|
|
2475
|
+
"attempt is the worst moment to fail."
|
|
2476
|
+
)
|
|
2477
|
+
report((again.stderr or again.stdout).strip())
|
|
2478
|
+
return 1
|
|
2479
|
+
if "removed=0" not in again.stdout:
|
|
2480
|
+
report(
|
|
2481
|
+
f"{label} M4 a second uninstall did not report that it removed "
|
|
2482
|
+
"nothing, so a run that cleaned nothing reads like a run that "
|
|
2483
|
+
"cleaned everything."
|
|
2484
|
+
)
|
|
2485
|
+
report((again.stderr or again.stdout).strip())
|
|
2486
|
+
return 1
|
|
2487
|
+
|
|
2488
|
+
bare = tmp / "install-only"
|
|
2489
|
+
bare.mkdir()
|
|
2490
|
+
bare_install = run_keel(bare, "--install", "--target", "claude")
|
|
2491
|
+
if bare_install.returncode != 0:
|
|
2492
|
+
report(f"{label} M4 install-only fixture failed.")
|
|
2493
|
+
report((bare_install.stderr or bare_install.stdout).strip())
|
|
2494
|
+
return 1
|
|
2495
|
+
bare_uninstall = run_keel(bare, "--uninstall", "--target", "claude")
|
|
2496
|
+
if bare_uninstall.returncode != 0:
|
|
2497
|
+
report(
|
|
2498
|
+
f"{label} M4 uninstalling a repository whose OpenSpec surfaces "
|
|
2499
|
+
"were never created failed. An absent surface is nothing to "
|
|
2500
|
+
"remove, not an error."
|
|
2501
|
+
)
|
|
2502
|
+
report((bare_uninstall.stderr or bare_uninstall.stdout).strip())
|
|
2503
|
+
return 1
|
|
2504
|
+
|
|
2505
|
+
if label not in {name for name, _ in SCENARIOS}:
|
|
2506
|
+
report(f"{label}: the scenario registry does not include it.")
|
|
2507
|
+
return 1
|
|
2508
|
+
report(f"{label} scenario passed.")
|
|
2509
|
+
return 0
|
|
2510
|
+
|
|
2511
|
+
|
|
2150
2512
|
def validate_sync_surface_overlay_scenario() -> int:
|
|
2151
2513
|
"""The surface that performs a gated action never named its gate.
|
|
2152
2514
|
|
|
@@ -2253,11 +2615,11 @@ def validate_sync_surface_overlay_scenario() -> int:
|
|
|
2253
2615
|
)
|
|
2254
2616
|
return 1
|
|
2255
2617
|
|
|
2256
|
-
# Uninstall is
|
|
2257
|
-
#
|
|
2258
|
-
#
|
|
2259
|
-
#
|
|
2260
|
-
#
|
|
2618
|
+
# Uninstall is asserted by `uninstall-removes-the-overlay`, which
|
|
2619
|
+
# covers every surface on every target rather than sync alone. It
|
|
2620
|
+
# stripped nothing from any surface when this scenario was written;
|
|
2621
|
+
# that was #50, and the assertion lives there because the contract
|
|
2622
|
+
# is about all four actions, not about this one.
|
|
2261
2623
|
|
|
2262
2624
|
# M4 — covering the surface did not widen what may be standing-authorized.
|
|
2263
2625
|
config_doc = (ROOT / "keel/config.yaml").read_text(encoding="utf-8")
|
|
@@ -2276,6 +2638,19 @@ def validate_sync_surface_overlay_scenario() -> int:
|
|
|
2276
2638
|
return 0
|
|
2277
2639
|
|
|
2278
2640
|
|
|
2641
|
+
def overlay_action_label(text: str, pattern: str) -> str | None:
|
|
2642
|
+
"""The action list a summary line names, or None when the line is absent.
|
|
2643
|
+
|
|
2644
|
+
`\\S*` rather than `\\S+` on purpose: a label that derived to nothing leaves
|
|
2645
|
+
two spaces where the actions belong, and matching it is what lets an empty
|
|
2646
|
+
label be reported as empty instead of as a missing line.
|
|
2647
|
+
"""
|
|
2648
|
+
match = re.search(pattern, text)
|
|
2649
|
+
if match is None:
|
|
2650
|
+
return None
|
|
2651
|
+
return match.group(1)
|
|
2652
|
+
|
|
2653
|
+
|
|
2279
2654
|
def validate_openspec_surface_overlay_scenario() -> int:
|
|
2280
2655
|
with tempfile.TemporaryDirectory(prefix="keel-overlay-") as raw_tmp:
|
|
2281
2656
|
tmp = Path(raw_tmp)
|
|
@@ -2300,6 +2675,61 @@ def validate_openspec_surface_overlay_scenario() -> int:
|
|
|
2300
2675
|
report((claude_doctor.stderr or claude_doctor.stdout).strip())
|
|
2301
2676
|
return 1
|
|
2302
2677
|
|
|
2678
|
+
# Every direction that reports the overlay names the actions it covers,
|
|
2679
|
+
# and all of them describe one surface list. They are compared against
|
|
2680
|
+
# each other rather than against a literal repeated here: the doctor
|
|
2681
|
+
# label is already pinned above, so an action joining the managed set
|
|
2682
|
+
# moves all three lines together or fails on this comparison. A third
|
|
2683
|
+
# copy of the string is what produced the defect this guards.
|
|
2684
|
+
#
|
|
2685
|
+
# The refresh line was the copy nobody was watching. Measured 2026-08-03
|
|
2686
|
+
# on one repository, back to back: `--init` reported `apply/archive`
|
|
2687
|
+
# while `--uninstall` reported `apply/archive/sync` (issue #75).
|
|
2688
|
+
#
|
|
2689
|
+
# Absence is checked before agreement, and separately, because two
|
|
2690
|
+
# labels that were never found also agree.
|
|
2691
|
+
claude_uninstall = run_keel(claude_repo, "--uninstall", "--target", "claude")
|
|
2692
|
+
if claude_uninstall.returncode != 0:
|
|
2693
|
+
report("openspec-surface-overlay scenario Claude uninstall failed:")
|
|
2694
|
+
report((claude_uninstall.stderr or claude_uninstall.stdout).strip())
|
|
2695
|
+
return 1
|
|
2696
|
+
summaries = {
|
|
2697
|
+
"refresh": overlay_action_label(
|
|
2698
|
+
claude_init.stdout, r"OpenSpec (\S*) overlay refreshed="
|
|
2699
|
+
),
|
|
2700
|
+
"doctor": overlay_action_label(
|
|
2701
|
+
claude_doctor.stdout, r"Keel (\S*) overlay:"
|
|
2702
|
+
),
|
|
2703
|
+
"removal": overlay_action_label(
|
|
2704
|
+
claude_uninstall.stdout, r"OpenSpec (\S*) overlay removed="
|
|
2705
|
+
),
|
|
2706
|
+
}
|
|
2707
|
+
for direction, printed in summaries.items():
|
|
2708
|
+
if printed is None:
|
|
2709
|
+
report(
|
|
2710
|
+
f"openspec-surface-overlay scenario found no {direction} "
|
|
2711
|
+
"overlay summary at all, so there was no label to compare "
|
|
2712
|
+
"rather than a label that disagreed."
|
|
2713
|
+
)
|
|
2714
|
+
return 1
|
|
2715
|
+
if not printed:
|
|
2716
|
+
report(
|
|
2717
|
+
f"openspec-surface-overlay scenario read an empty action "
|
|
2718
|
+
f"label from the {direction} summary; a label that derives "
|
|
2719
|
+
"to nothing reports no actions at all."
|
|
2720
|
+
)
|
|
2721
|
+
return 1
|
|
2722
|
+
for direction in ("refresh", "removal"):
|
|
2723
|
+
if summaries[direction] != summaries["doctor"]:
|
|
2724
|
+
report(
|
|
2725
|
+
"openspec-surface-overlay scenario overlay summaries "
|
|
2726
|
+
f"disagree: {direction} names {summaries[direction]!r} and "
|
|
2727
|
+
f"doctor names {summaries['doctor']!r}. They describe the "
|
|
2728
|
+
"same managed surface list, so one of them is written "
|
|
2729
|
+
"beside the action set instead of derived from it."
|
|
2730
|
+
)
|
|
2731
|
+
return 1
|
|
2732
|
+
|
|
2303
2733
|
codex_repo = tmp / "codex"
|
|
2304
2734
|
codex_repo.mkdir()
|
|
2305
2735
|
codex_home = tmp / "codex-home"
|
|
@@ -2762,6 +3192,21 @@ def validate_cli_scenario() -> int:
|
|
|
2762
3192
|
report((help_result.stderr or help_result.stdout).strip())
|
|
2763
3193
|
return 1
|
|
2764
3194
|
|
|
3195
|
+
usage_block = help_result.stdout.split("Usage:", 1)[1].split("\n\n", 1)[0]
|
|
3196
|
+
triage_lines = [line for line in usage_block.splitlines() if "keel triage" in line]
|
|
3197
|
+
if not triage_lines:
|
|
3198
|
+
report("cli scenario expected keel --help Usage: block to list a `keel triage` line.")
|
|
3199
|
+
report(usage_block.strip())
|
|
3200
|
+
return 1
|
|
3201
|
+
if "--labels" not in triage_lines[0]:
|
|
3202
|
+
report("cli scenario expected the keel --help `keel triage` line to name --labels.")
|
|
3203
|
+
report(triage_lines[0].strip())
|
|
3204
|
+
return 1
|
|
3205
|
+
if "--issue" not in triage_lines[0]:
|
|
3206
|
+
report("cli scenario expected the keel --help `keel triage` line to name --issue.")
|
|
3207
|
+
report(triage_lines[0].strip())
|
|
3208
|
+
return 1
|
|
3209
|
+
|
|
2765
3210
|
version_result = run_keel(ROOT, "--version")
|
|
2766
3211
|
expected_version = f"keel {PACKAGE_VERSION}"
|
|
2767
3212
|
if version_result.returncode != 0 or version_result.stdout.strip() != expected_version:
|
|
@@ -3837,55 +4282,368 @@ def validate_task_start_invalidation_scenario() -> int:
|
|
|
3837
4282
|
return 0
|
|
3838
4283
|
|
|
3839
4284
|
|
|
3840
|
-
|
|
3841
|
-
|
|
3842
|
-
|
|
3843
|
-
|
|
3844
|
-
|
|
3845
|
-
|
|
3846
|
-
|
|
3847
|
-
|
|
3848
|
-
|
|
3849
|
-
)
|
|
4285
|
+
# A task for the section-boundary fixtures. Two shapes matter and nothing else
|
|
4286
|
+
# does: `covers` puts `- E<n>:` lines inside a task body, which a section that
|
|
4287
|
+
# ran over the task list read as its own entries, and `repo_action` gives the
|
|
4288
|
+
# task a `Touch` of a bare `- none`, which such a section read as its `- None.`
|
|
4289
|
+
# and closed itself with. Every other field is the cheapest legitimate value.
|
|
4290
|
+
def section_boundary_task(
|
|
4291
|
+
task_id: str,
|
|
4292
|
+
*,
|
|
4293
|
+
checked: bool,
|
|
4294
|
+
covers: tuple[str, ...] = ("E1: the task owns its file",),
|
|
4295
|
+
repo_action: bool = False,
|
|
4296
|
+
) -> str:
|
|
4297
|
+
covers_lines = "".join(f" - {item}\n" for item in covers)
|
|
4298
|
+
touch = " - none\n" if repo_action else " - src/example.js\n"
|
|
4299
|
+
mode = "repo-action" if repo_action else "implementation"
|
|
4300
|
+
return (
|
|
4301
|
+
f"- [{'x' if checked else ' '}] {task_id} Section boundary fixture\n"
|
|
4302
|
+
" - Owner: claude\n"
|
|
4303
|
+
f" - Mode: {mode}\n"
|
|
4304
|
+
" - Covers:\n"
|
|
4305
|
+
f"{covers_lines}"
|
|
4306
|
+
" - Read:\n"
|
|
4307
|
+
" - README.md\n"
|
|
4308
|
+
" - Touch:\n"
|
|
4309
|
+
f"{touch}"
|
|
4310
|
+
" - Verify:\n"
|
|
4311
|
+
" - Strategy: evidence-first\n"
|
|
4312
|
+
" - M1: node test.js reports the public behavior passing\n"
|
|
4313
|
+
" - Acceptance:\n"
|
|
4314
|
+
" - Public behavior passes.\n"
|
|
4315
|
+
" - Autonomy boundary:\n"
|
|
4316
|
+
" - Default: hard-stop\n"
|
|
4317
|
+
" - Pre-authorized fallback: none\n"
|
|
4318
|
+
" - Stop Rules:\n"
|
|
4319
|
+
" - Stop on failure.\n"
|
|
4320
|
+
" - Evidence:\n"
|
|
4321
|
+
" - Contract: pending\n"
|
|
4322
|
+
" - M1: node test.js reported 4 passing and 0 failing\n"
|
|
4323
|
+
" - Review:\n"
|
|
4324
|
+
" - Status: pass\n"
|
|
4325
|
+
" - Acceptance check: the public behavior was exercised\n"
|
|
4326
|
+
" - Scope check: only the declared file changed\n"
|
|
4327
|
+
" - Findings: none\n"
|
|
4328
|
+
" - Blocker: none\n"
|
|
4329
|
+
" - Stop if:\n"
|
|
4330
|
+
" - Requires files outside Touch.\n"
|
|
4331
|
+
)
|
|
3850
4332
|
|
|
3851
4333
|
|
|
3852
|
-
|
|
3853
|
-
|
|
4334
|
+
# The same file with the section above the task list and in the tail. Nothing
|
|
4335
|
+
# else differs between the two, which is what makes the pair a measurement of
|
|
4336
|
+
# position rather than of content.
|
|
4337
|
+
def section_boundary_tasks_md(*, section: str, task: str, position: str) -> str:
|
|
4338
|
+
if position == "above":
|
|
4339
|
+
return "# Tasks\n\n" + section + "\n" + task
|
|
4340
|
+
return "# Tasks\n\n" + task + "\n" + section
|
|
3854
4341
|
|
|
3855
|
-
# A dry run is relied on, so it is wrong in both directions: naming a write
|
|
3856
|
-
# that will not happen trains the reader to ignore it, and omitting one
|
|
3857
|
-
# breaks the promise the dry run exists to make. `--check` used to omit the
|
|
3858
|
-
# overlay step entirely while `--install --dry-run` claimed every surface.
|
|
3859
|
-
def overlay_lines(text: str) -> list[str]:
|
|
3860
|
-
return [line for line in text.splitlines() if "overlay" in line]
|
|
3861
4342
|
|
|
3862
|
-
|
|
3863
|
-
|
|
3864
|
-
|
|
4343
|
+
def section_boundary_repo(root: Path, name: str, content: str) -> Path:
|
|
4344
|
+
repo = root / name
|
|
4345
|
+
change = repo / "openspec/changes/demo"
|
|
4346
|
+
write_text(change / "tasks.md", content)
|
|
4347
|
+
write_text(change / "proposal.md", "# Proposal\n")
|
|
4348
|
+
write_text(change / "design.md", "## Context\n\nfixture\n")
|
|
4349
|
+
write_text(change / "specs/demo/spec.md", "## ADDED Requirements\n")
|
|
4350
|
+
return repo
|
|
3865
4351
|
|
|
3866
|
-
with tempfile.TemporaryDirectory(prefix="keel-dry-run-overlay-") as raw_tmp:
|
|
3867
|
-
repo = Path(raw_tmp) / "repo"
|
|
3868
|
-
repo.mkdir()
|
|
3869
|
-
# The overlay surfaces are files OpenSpec generates; install merges into
|
|
3870
|
-
# them and skips the ones that are absent. Create them so this scenario
|
|
3871
|
-
# exercises the classification rather than the missing branch.
|
|
3872
|
-
for relative in (
|
|
3873
|
-
".claude/skills/openspec-propose/SKILL.md",
|
|
3874
|
-
".claude/skills/openspec-apply-change/SKILL.md",
|
|
3875
|
-
".claude/skills/openspec-archive-change/SKILL.md",
|
|
3876
|
-
".claude/commands/opsx/propose.md",
|
|
3877
|
-
".claude/commands/opsx/apply.md",
|
|
3878
|
-
".claude/commands/opsx/archive.md",
|
|
3879
|
-
):
|
|
3880
|
-
write_text(repo / relative, "# OpenSpec surface\n\nGenerated body.\n")
|
|
3881
|
-
if run_keel(repo, "--install", "--target", "claude").returncode != 0:
|
|
3882
|
-
report(f"{label} could not install a fixture repository.")
|
|
3883
|
-
return 1
|
|
3884
4352
|
|
|
3885
|
-
|
|
3886
|
-
|
|
3887
|
-
|
|
3888
|
-
|
|
4353
|
+
def validate_section_boundary_scenario() -> int:
|
|
4354
|
+
"""A change-level section is bounded by the tasks, not only by headings.
|
|
4355
|
+
|
|
4356
|
+
Regression for issue #71 and for the silent half the issue does not report.
|
|
4357
|
+
Both readers of a change-level section sliced to the next `## ` heading, so
|
|
4358
|
+
a section that was not the file's last one ran over the task list: the
|
|
4359
|
+
`- E<n>:` lines a task declares under `Covers` were judged as coverage
|
|
4360
|
+
entries and reported unclosed, and a `repo-action` task's `Touch` of a bare
|
|
4361
|
+
`- none` was read as the section's `- None.` and closed a declaration that
|
|
4362
|
+
closed nothing.
|
|
4363
|
+
|
|
4364
|
+
Every cell is a pair — the identical section above the task list and in the
|
|
4365
|
+
tail — asserted both absolutely and against each other. The absolute half
|
|
4366
|
+
matters: a comparison alone passes when both positions are broken the same
|
|
4367
|
+
way.
|
|
4368
|
+
"""
|
|
4369
|
+
label = "section-boundary"
|
|
4370
|
+
|
|
4371
|
+
def problems_of(payload: dict) -> list[tuple[str, str]]:
|
|
4372
|
+
return sorted(
|
|
4373
|
+
(item.get("code", ""), item.get("message", ""))
|
|
4374
|
+
for item in payload.get("problems", [])
|
|
4375
|
+
)
|
|
4376
|
+
|
|
4377
|
+
def closure_problems(payload: dict, code: str) -> list[str]:
|
|
4378
|
+
return [
|
|
4379
|
+
item.get("message", "")
|
|
4380
|
+
for item in payload.get("problems", [])
|
|
4381
|
+
if item.get("code") == code
|
|
4382
|
+
]
|
|
4383
|
+
|
|
4384
|
+
with tempfile.TemporaryDirectory(
|
|
4385
|
+
prefix="keel-section-boundary-", ignore_cleanup_errors=True
|
|
4386
|
+
) as raw:
|
|
4387
|
+
tmp = Path(raw)
|
|
4388
|
+
|
|
4389
|
+
# `None` and `{}` are kept apart on purpose. A fixture task-start
|
|
4390
|
+
# refused and a gate that printed nothing are different failures, and a
|
|
4391
|
+
# message naming one of them while the other happened sends the reader
|
|
4392
|
+
# to a place with no problem in it.
|
|
4393
|
+
def close(name: str, content: str) -> dict | None:
|
|
4394
|
+
repo = section_boundary_repo(tmp, name, content)
|
|
4395
|
+
# A checked task with no recorded anchor fails the close for a
|
|
4396
|
+
# reason that has nothing to do with this boundary, and that noise
|
|
4397
|
+
# would land in both halves of every pair.
|
|
4398
|
+
if not record_contract_anchor(repo, "demo", "1.1"):
|
|
4399
|
+
return None
|
|
4400
|
+
result = run_keel(
|
|
4401
|
+
repo, "gate", "change-close", ".",
|
|
4402
|
+
"--change", "demo", "--action", "archive", "--json",
|
|
4403
|
+
)
|
|
4404
|
+
return json.loads(result.stdout) if result.stdout.strip() else {}
|
|
4405
|
+
|
|
4406
|
+
def start(name: str, content: str) -> dict:
|
|
4407
|
+
repo = section_boundary_repo(tmp, name, content)
|
|
4408
|
+
result = run_keel(
|
|
4409
|
+
repo, "gate", "task-start", ".",
|
|
4410
|
+
"--change", "demo", "--task", "1.1", "--no-guard", "--json",
|
|
4411
|
+
)
|
|
4412
|
+
return json.loads(result.stdout) if result.stdout.strip() else {}
|
|
4413
|
+
|
|
4414
|
+
def readable(payload: dict | None, fixture: str, gate: str) -> bool:
|
|
4415
|
+
if payload is None:
|
|
4416
|
+
report(
|
|
4417
|
+
f"{label} could not record a Contract anchor for {fixture}: "
|
|
4418
|
+
"task-start refused the fixture, so change-close never ran "
|
|
4419
|
+
"against it."
|
|
4420
|
+
)
|
|
4421
|
+
return False
|
|
4422
|
+
if not payload:
|
|
4423
|
+
report(
|
|
4424
|
+
f"{label} got no JSON from {gate} for {fixture}, so there "
|
|
4425
|
+
"was no verdict to read rather than a verdict that was "
|
|
4426
|
+
"wrong."
|
|
4427
|
+
)
|
|
4428
|
+
return False
|
|
4429
|
+
return True
|
|
4430
|
+
|
|
4431
|
+
# Cell 1: `## Expectation Coverage` closes every entry it declares, and
|
|
4432
|
+
# the task below it declares `- E1:`/`- E2:` under its own `Covers`.
|
|
4433
|
+
# Nothing here is unclosed in either position.
|
|
4434
|
+
closed_coverage = (
|
|
4435
|
+
"## Invalidates\n\n- None.\n\n"
|
|
4436
|
+
"## Expectation Coverage\n\n"
|
|
4437
|
+
"- E1: The task owns its file. Covered by: 1.1\n"
|
|
4438
|
+
"- E2: The task proves it. Covered by: 1.1\n"
|
|
4439
|
+
)
|
|
4440
|
+
closed_task = section_boundary_task(
|
|
4441
|
+
"1.1",
|
|
4442
|
+
checked=True,
|
|
4443
|
+
covers=("E1: The task owns its file.", "E2: The task proves it."),
|
|
4444
|
+
)
|
|
4445
|
+
closed = {
|
|
4446
|
+
position: close(
|
|
4447
|
+
f"coverage-closed-{position}",
|
|
4448
|
+
section_boundary_tasks_md(
|
|
4449
|
+
section=closed_coverage, task=closed_task, position=position
|
|
4450
|
+
),
|
|
4451
|
+
)
|
|
4452
|
+
for position in ("above", "tail")
|
|
4453
|
+
}
|
|
4454
|
+
for position, payload in closed.items():
|
|
4455
|
+
if not readable(payload, f"coverage-closed-{position}", "change-close"):
|
|
4456
|
+
return 1
|
|
4457
|
+
stray = closure_problems(payload, "expectation-closure")
|
|
4458
|
+
if stray:
|
|
4459
|
+
report(
|
|
4460
|
+
f"{label} reported an expectation the section closes as "
|
|
4461
|
+
f"unclosed, with the section {position} the task list. The "
|
|
4462
|
+
"entries it judged are the task's own Covers lines."
|
|
4463
|
+
)
|
|
4464
|
+
report(repr(stray))
|
|
4465
|
+
return 1
|
|
4466
|
+
if problems_of(closed["above"]) != problems_of(closed["tail"]):
|
|
4467
|
+
report(
|
|
4468
|
+
f"{label} returned different problems for identical section "
|
|
4469
|
+
"content depending on where the section sits."
|
|
4470
|
+
)
|
|
4471
|
+
report(repr(problems_of(closed["above"])))
|
|
4472
|
+
report(repr(problems_of(closed["tail"])))
|
|
4473
|
+
return 1
|
|
4474
|
+
if closed["tail"].get("status") != "pass":
|
|
4475
|
+
report(
|
|
4476
|
+
f"{label} refused a fixture whose only expectations are closed; "
|
|
4477
|
+
"the pair comparison above would then be comparing two failures."
|
|
4478
|
+
)
|
|
4479
|
+
report(repr(closed["tail"].get("problems")))
|
|
4480
|
+
return 1
|
|
4481
|
+
|
|
4482
|
+
# Cell 2: the silent half. The entry closes nothing, and the task below
|
|
4483
|
+
# is a `repo-action` whose `Touch` is a bare `- none`.
|
|
4484
|
+
open_coverage = (
|
|
4485
|
+
"## Invalidates\n\n- None.\n\n"
|
|
4486
|
+
"## Expectation Coverage\n\n"
|
|
4487
|
+
"- E3: Nothing closes this one.\n"
|
|
4488
|
+
)
|
|
4489
|
+
none_task = section_boundary_task("1.1", checked=True, repo_action=True)
|
|
4490
|
+
unclosed = {
|
|
4491
|
+
position: close(
|
|
4492
|
+
f"coverage-open-{position}",
|
|
4493
|
+
section_boundary_tasks_md(
|
|
4494
|
+
section=open_coverage, task=none_task, position=position
|
|
4495
|
+
),
|
|
4496
|
+
)
|
|
4497
|
+
for position in ("above", "tail")
|
|
4498
|
+
}
|
|
4499
|
+
for position, payload in unclosed.items():
|
|
4500
|
+
if not readable(payload, f"coverage-open-{position}", "change-close"):
|
|
4501
|
+
return 1
|
|
4502
|
+
named = [
|
|
4503
|
+
message
|
|
4504
|
+
for message in closure_problems(payload, "expectation-closure")
|
|
4505
|
+
if "E3" in message
|
|
4506
|
+
]
|
|
4507
|
+
if not named:
|
|
4508
|
+
report(
|
|
4509
|
+
f"{label} accepted an expectation that closes nothing, with "
|
|
4510
|
+
f"the section {position} the task list. A task field of "
|
|
4511
|
+
"`none` is not the section's `- None.`."
|
|
4512
|
+
)
|
|
4513
|
+
report(repr(payload.get("problems")))
|
|
4514
|
+
return 1
|
|
4515
|
+
if problems_of(unclosed["above"]) != problems_of(unclosed["tail"]):
|
|
4516
|
+
report(
|
|
4517
|
+
f"{label} returned different problems for an identical unclosed "
|
|
4518
|
+
"section depending on where the section sits."
|
|
4519
|
+
)
|
|
4520
|
+
report(repr(problems_of(unclosed["above"])))
|
|
4521
|
+
report(repr(problems_of(unclosed["tail"])))
|
|
4522
|
+
return 1
|
|
4523
|
+
|
|
4524
|
+
# Cell 3: the other reader. `## Invalidates` is read by task-start, and
|
|
4525
|
+
# holds the character-identical slice.
|
|
4526
|
+
closed_invalidates = (
|
|
4527
|
+
"## Invalidates\n\n"
|
|
4528
|
+
'- I1: "the wording that is now wrong" — README.md. Updated by: 1.1\n'
|
|
4529
|
+
)
|
|
4530
|
+
start_task = section_boundary_task("1.1", checked=False)
|
|
4531
|
+
started = {
|
|
4532
|
+
position: start(
|
|
4533
|
+
f"invalidates-closed-{position}",
|
|
4534
|
+
section_boundary_tasks_md(
|
|
4535
|
+
section=closed_invalidates, task=start_task, position=position
|
|
4536
|
+
),
|
|
4537
|
+
)
|
|
4538
|
+
for position in ("above", "tail")
|
|
4539
|
+
}
|
|
4540
|
+
for position, payload in started.items():
|
|
4541
|
+
if not readable(payload, f"invalidates-closed-{position}", "task-start"):
|
|
4542
|
+
return 1
|
|
4543
|
+
if payload.get("status") != "pass":
|
|
4544
|
+
report(
|
|
4545
|
+
f"{label} refused a closed invalidation declaration with "
|
|
4546
|
+
f"the section {position} the task list."
|
|
4547
|
+
)
|
|
4548
|
+
report(repr(payload.get("problems")))
|
|
4549
|
+
return 1
|
|
4550
|
+
|
|
4551
|
+
# Cell 4: the silent half of the other reader. This is the cell that
|
|
4552
|
+
# returned `pass` before the boundary gained its task half.
|
|
4553
|
+
open_invalidates = (
|
|
4554
|
+
"## Invalidates\n\n"
|
|
4555
|
+
'- I1: "the wording that is now wrong" — README.md.\n'
|
|
4556
|
+
)
|
|
4557
|
+
open_start_task = section_boundary_task(
|
|
4558
|
+
"1.1", checked=False, repo_action=True
|
|
4559
|
+
)
|
|
4560
|
+
unstarted = {
|
|
4561
|
+
position: start(
|
|
4562
|
+
f"invalidates-open-{position}",
|
|
4563
|
+
section_boundary_tasks_md(
|
|
4564
|
+
section=open_invalidates, task=open_start_task, position=position
|
|
4565
|
+
),
|
|
4566
|
+
)
|
|
4567
|
+
for position in ("above", "tail")
|
|
4568
|
+
}
|
|
4569
|
+
for position, payload in unstarted.items():
|
|
4570
|
+
if not readable(payload, f"invalidates-open-{position}", "task-start"):
|
|
4571
|
+
return 1
|
|
4572
|
+
named = [
|
|
4573
|
+
message
|
|
4574
|
+
for message in closure_problems(payload, "invalidation-closure")
|
|
4575
|
+
if "I1" in message
|
|
4576
|
+
]
|
|
4577
|
+
if not named:
|
|
4578
|
+
report(
|
|
4579
|
+
f"{label} accepted an invalidation that closes nothing, "
|
|
4580
|
+
f"with the section {position} the task list. A task field "
|
|
4581
|
+
"of `none` is not the section's `- None.`."
|
|
4582
|
+
)
|
|
4583
|
+
report(repr(payload.get("problems")))
|
|
4584
|
+
return 1
|
|
4585
|
+
if problems_of(unstarted["above"]) != problems_of(unstarted["tail"]):
|
|
4586
|
+
report(
|
|
4587
|
+
f"{label} returned different problems for an identical unclosed "
|
|
4588
|
+
"invalidation depending on where the section sits."
|
|
4589
|
+
)
|
|
4590
|
+
report(repr(problems_of(unstarted["above"])))
|
|
4591
|
+
report(repr(problems_of(unstarted["tail"])))
|
|
4592
|
+
return 1
|
|
4593
|
+
|
|
4594
|
+
report(f"{label} scenario passed.")
|
|
4595
|
+
return 0
|
|
4596
|
+
|
|
4597
|
+
|
|
4598
|
+
SCHEMA_COPY_PAIRS = (
|
|
4599
|
+
(
|
|
4600
|
+
"openspec/schemas/keel-spec-driven/templates/tasks.md",
|
|
4601
|
+
"assets/openspec/schemas/keel-spec-driven/templates/tasks.md",
|
|
4602
|
+
),
|
|
4603
|
+
(
|
|
4604
|
+
"openspec/schemas/keel-spec-driven/schema.yaml",
|
|
4605
|
+
"assets/openspec/schemas/keel-spec-driven/schema.yaml",
|
|
4606
|
+
),
|
|
4607
|
+
)
|
|
4608
|
+
|
|
4609
|
+
|
|
4610
|
+
def validate_dry_run_overlay_accounting_scenario() -> int:
|
|
4611
|
+
label = "dry-run-overlay-accounting"
|
|
4612
|
+
|
|
4613
|
+
# A dry run is relied on, so it is wrong in both directions: naming a write
|
|
4614
|
+
# that will not happen trains the reader to ignore it, and omitting one
|
|
4615
|
+
# breaks the promise the dry run exists to make. `--check` used to omit the
|
|
4616
|
+
# overlay step entirely while `--install --dry-run` claimed every surface.
|
|
4617
|
+
def overlay_lines(text: str) -> list[str]:
|
|
4618
|
+
return [line for line in text.splitlines() if "overlay" in line]
|
|
4619
|
+
|
|
4620
|
+
def counts(text: str) -> str | None:
|
|
4621
|
+
found = re.search(r"refreshed=(\d+) current=(\d+) missing=(\d+)", text)
|
|
4622
|
+
return found.group(0) if found else None
|
|
4623
|
+
|
|
4624
|
+
with tempfile.TemporaryDirectory(prefix="keel-dry-run-overlay-") as raw_tmp:
|
|
4625
|
+
repo = Path(raw_tmp) / "repo"
|
|
4626
|
+
repo.mkdir()
|
|
4627
|
+
# The overlay surfaces are files OpenSpec generates; install merges into
|
|
4628
|
+
# them and skips the ones that are absent. Create them so this scenario
|
|
4629
|
+
# exercises the classification rather than the missing branch.
|
|
4630
|
+
for relative in (
|
|
4631
|
+
".claude/skills/openspec-propose/SKILL.md",
|
|
4632
|
+
".claude/skills/openspec-apply-change/SKILL.md",
|
|
4633
|
+
".claude/skills/openspec-archive-change/SKILL.md",
|
|
4634
|
+
".claude/commands/opsx/propose.md",
|
|
4635
|
+
".claude/commands/opsx/apply.md",
|
|
4636
|
+
".claude/commands/opsx/archive.md",
|
|
4637
|
+
):
|
|
4638
|
+
write_text(repo / relative, "# OpenSpec surface\n\nGenerated body.\n")
|
|
4639
|
+
if run_keel(repo, "--install", "--target", "claude").returncode != 0:
|
|
4640
|
+
report(f"{label} could not install a fixture repository.")
|
|
4641
|
+
return 1
|
|
4642
|
+
|
|
4643
|
+
# Nothing stale: neither dry run may name a file.
|
|
4644
|
+
check = run_keel(repo, "--check", "--target", "claude")
|
|
4645
|
+
if any("would refresh OpenSpec" in line and ".md" in line
|
|
4646
|
+
for line in overlay_lines(check.stdout)):
|
|
3889
4647
|
report(f"{label} named a surface that would not change.")
|
|
3890
4648
|
report("\n".join(overlay_lines(check.stdout)))
|
|
3891
4649
|
return 1
|
|
@@ -4424,84 +5182,357 @@ def validate_regression_check_tag_scenario() -> int:
|
|
|
4424
5182
|
return 0
|
|
4425
5183
|
|
|
4426
5184
|
|
|
4427
|
-
def
|
|
4428
|
-
|
|
5185
|
+
def change_verify_task(
|
|
5186
|
+
task_id: str,
|
|
5187
|
+
*,
|
|
5188
|
+
checked: bool,
|
|
5189
|
+
commands: tuple[str, ...],
|
|
5190
|
+
evidence: tuple[str, ...],
|
|
5191
|
+
) -> str:
|
|
5192
|
+
command_lines = "".join(f" - {item}\n" for item in commands)
|
|
5193
|
+
evidence_lines = "".join(f" - {item}\n" for item in evidence)
|
|
5194
|
+
return (
|
|
5195
|
+
f"- [{'x' if checked else ' '}] {task_id} Change verify fixture\n"
|
|
5196
|
+
" - Owner: claude\n"
|
|
5197
|
+
" - Mode: implementation\n"
|
|
5198
|
+
" - Covers:\n"
|
|
5199
|
+
" - E1: the task proves its own behavior\n"
|
|
5200
|
+
" - Read:\n"
|
|
5201
|
+
" - README.md\n"
|
|
5202
|
+
" - Touch:\n"
|
|
5203
|
+
" - src/example.js\n"
|
|
5204
|
+
" - Verify:\n"
|
|
5205
|
+
" - Strategy: vertical-tdd\n"
|
|
5206
|
+
f"{command_lines}"
|
|
5207
|
+
" - Acceptance:\n"
|
|
5208
|
+
" - Public behavior passes.\n"
|
|
5209
|
+
" - Autonomy boundary:\n"
|
|
5210
|
+
" - Default: hard-stop\n"
|
|
5211
|
+
" - Pre-authorized fallback: none\n"
|
|
5212
|
+
" - Stop Rules:\n"
|
|
5213
|
+
" - Stop on failure.\n"
|
|
5214
|
+
" - Evidence:\n"
|
|
5215
|
+
" - Contract: pending\n"
|
|
5216
|
+
f"{evidence_lines}"
|
|
5217
|
+
" - Review:\n"
|
|
5218
|
+
" - Status: pass\n"
|
|
5219
|
+
" - Acceptance check: behavior proven.\n"
|
|
5220
|
+
" - Scope check: writes stayed inside Touch.\n"
|
|
5221
|
+
" - Findings: none\n"
|
|
5222
|
+
" - Blocker: none\n"
|
|
5223
|
+
" - Stop if:\n"
|
|
5224
|
+
" - Requires files outside Touch.\n"
|
|
5225
|
+
)
|
|
4429
5226
|
|
|
4430
|
-
# The helper derives the consumer-repo paths every install/uninstall/clear
|
|
4431
|
-
# assertion iterates. When its root stopped existing it returned an empty
|
|
4432
|
-
# list, so those loops compared nothing and reported success. Anchor it to
|
|
4433
|
-
# what the installer really writes, and make emptiness a failure here.
|
|
4434
|
-
try:
|
|
4435
|
-
packaged_openspec_schema_install_paths(ROOT / "no-such-packaged-root")
|
|
4436
|
-
except FileNotFoundError as error:
|
|
4437
|
-
if "no-such-packaged-root" not in str(error):
|
|
4438
|
-
report(f"{label} missing-root failure does not name the path it expected.")
|
|
4439
|
-
report(str(error))
|
|
4440
|
-
return 1
|
|
4441
|
-
else:
|
|
4442
|
-
report(
|
|
4443
|
-
f"{label} returned a set for a missing packaged root instead of failing; "
|
|
4444
|
-
"an absent root must not silently empty its callers' assertions."
|
|
4445
|
-
)
|
|
4446
|
-
return 1
|
|
4447
5227
|
|
|
4448
|
-
|
|
4449
|
-
|
|
4450
|
-
|
|
4451
|
-
|
|
4452
|
-
|
|
4453
|
-
|
|
4454
|
-
return 1
|
|
5228
|
+
def change_verify_deferred_evidence_repo(root: Path, name: str) -> Path:
|
|
5229
|
+
repo = root / name
|
|
5230
|
+
write_text(repo / "openspec/changes/demo/proposal.md", "# Proposal\n")
|
|
5231
|
+
write_text(repo / "openspec/changes/demo/design.md", "## Context\n\nfixture\n")
|
|
5232
|
+
write_text(repo / "openspec/changes/demo/specs/demo/spec.md", "## ADDED Requirements\n")
|
|
5233
|
+
return repo
|
|
4455
5234
|
|
|
4456
|
-
with tempfile.TemporaryDirectory(prefix="keel-packaged-schema-") as raw_tmp:
|
|
4457
|
-
repo = Path(raw_tmp) / "repo"
|
|
4458
|
-
repo.mkdir()
|
|
4459
|
-
install = run_keel(repo, "--install")
|
|
4460
|
-
if install.returncode != 0:
|
|
4461
|
-
report(f"{label} keel --install failed.")
|
|
4462
|
-
report((install.stderr or install.stdout).strip())
|
|
4463
|
-
return 1
|
|
4464
5235
|
|
|
4465
|
-
|
|
4466
|
-
|
|
4467
|
-
|
|
4468
|
-
|
|
4469
|
-
|
|
4470
|
-
|
|
4471
|
-
|
|
4472
|
-
|
|
4473
|
-
f"{label} derived paths do not match what keel --install wrote."
|
|
4474
|
-
)
|
|
4475
|
-
report(f"derived: {sorted(derived)}")
|
|
4476
|
-
report(f"installed: {installed}")
|
|
4477
|
-
return 1
|
|
5236
|
+
def validate_change_verify_deferred_evidence_scenario() -> int:
|
|
5237
|
+
"""A `(regression)`-tagged check may defer its bare Evidence to a
|
|
5238
|
+
change-level `C<n>` check, run once for the whole change instead of once
|
|
5239
|
+
per task. Issue #95: a full regression suite that only needs to run once
|
|
5240
|
+
had no place to live in `tasks.md`, so authors either paid to repeat it on
|
|
5241
|
+
every task or gave the middle tasks no safety net at all.
|
|
5242
|
+
"""
|
|
5243
|
+
label = "change-verify-deferred-evidence"
|
|
4478
5244
|
|
|
4479
|
-
|
|
4480
|
-
|
|
5245
|
+
change_verify_c1 = (
|
|
5246
|
+
"## Change Verify\n\n"
|
|
5247
|
+
"- Strategy: regression-first\n"
|
|
5248
|
+
"- C1: the full three-layer suite is 0 failed\n\n"
|
|
5249
|
+
)
|
|
5250
|
+
change_evidence_c1_pending = "## Change Evidence\n\n- C1: pending\n\n"
|
|
5251
|
+
change_evidence_c1_pass = (
|
|
5252
|
+
"## Change Evidence\n\n- C1: pass. Full suite ran clean at 142/142.\n\n"
|
|
5253
|
+
)
|
|
5254
|
+
tagged_commands = (
|
|
5255
|
+
"M1: behavior reaches the public interface",
|
|
5256
|
+
"M2 (regression): the full suite stays green",
|
|
5257
|
+
)
|
|
5258
|
+
deferred_evidence = (
|
|
5259
|
+
"M1: behavior exercised.",
|
|
5260
|
+
"M1.red: failed before the implementation.",
|
|
5261
|
+
"M1.green: passed after.",
|
|
5262
|
+
"M2: deferred to C1",
|
|
5263
|
+
)
|
|
4481
5264
|
|
|
5265
|
+
def tasks_md(
|
|
5266
|
+
*,
|
|
5267
|
+
change_verify: str = "",
|
|
5268
|
+
change_evidence: str = "",
|
|
5269
|
+
commands: tuple[str, ...] = tagged_commands,
|
|
5270
|
+
evidence: tuple[str, ...],
|
|
5271
|
+
checked: bool = True,
|
|
5272
|
+
) -> str:
|
|
5273
|
+
sections = (
|
|
5274
|
+
"## Invalidates\n\n- None.\n\n"
|
|
5275
|
+
"## Expectation Coverage\n\n"
|
|
5276
|
+
"- E1: the task proves its own behavior. Covered by: 1.1\n\n"
|
|
5277
|
+
+ change_verify
|
|
5278
|
+
+ change_evidence
|
|
5279
|
+
)
|
|
5280
|
+
return "# Tasks\n\n" + sections + change_verify_task(
|
|
5281
|
+
"1.1", checked=checked, commands=commands, evidence=evidence
|
|
5282
|
+
)
|
|
4482
5283
|
|
|
4483
|
-
|
|
4484
|
-
|
|
5284
|
+
with tempfile.TemporaryDirectory(
|
|
5285
|
+
prefix="keel-change-verify-", ignore_cleanup_errors=True
|
|
5286
|
+
) as raw:
|
|
5287
|
+
tmp = Path(raw)
|
|
5288
|
+
repo = change_verify_deferred_evidence_repo(tmp, "repo")
|
|
5289
|
+
tasks_path = repo / "openspec/changes/demo/tasks.md"
|
|
4485
5290
|
|
|
4486
|
-
|
|
4487
|
-
|
|
4488
|
-
# agree: compact-task-authoring used to imply it through a projection loop
|
|
4489
|
-
# rooted at trees that no longer exist, so it compared nothing and has since
|
|
4490
|
-
# been removed.
|
|
4491
|
-
for local, packaged in SCHEMA_COPY_PAIRS:
|
|
4492
|
-
local_text = (ROOT / local).read_text(encoding="utf-8")
|
|
4493
|
-
packaged_text = (ROOT / packaged).read_text(encoding="utf-8")
|
|
4494
|
-
if local_text != packaged_text:
|
|
4495
|
-
report(f"{label} schema copies diverge: {local} vs {packaged}")
|
|
4496
|
-
return 1
|
|
5291
|
+
def codes(payload: dict) -> set[str]:
|
|
5292
|
+
return {item.get("code") for item in payload.get("problems", [])}
|
|
4497
5293
|
|
|
4498
|
-
|
|
4499
|
-
|
|
4500
|
-
|
|
4501
|
-
|
|
4502
|
-
|
|
4503
|
-
|
|
4504
|
-
return
|
|
5294
|
+
def complete(fixture: str) -> dict:
|
|
5295
|
+
write_text(tasks_path, fixture)
|
|
5296
|
+
record_contract_anchor(repo, "demo")
|
|
5297
|
+
result = run_keel(
|
|
5298
|
+
repo, "gate", "task-complete", "--change", "demo", "--task", "1.1", "--json"
|
|
5299
|
+
)
|
|
5300
|
+
return json.loads(result.stdout)
|
|
5301
|
+
|
|
5302
|
+
def close(fixture: str) -> dict:
|
|
5303
|
+
write_text(tasks_path, fixture)
|
|
5304
|
+
record_contract_anchor(repo, "demo")
|
|
5305
|
+
result = run_keel(
|
|
5306
|
+
repo, "gate", "change-close", "--change", "demo", "--action", "sync", "--json"
|
|
5307
|
+
)
|
|
5308
|
+
return json.loads(result.stdout)
|
|
5309
|
+
|
|
5310
|
+
# (a) A `(regression)`-tagged check deferring to a declared C1 passes
|
|
5311
|
+
# task-complete with no C1 result required yet — the change has not
|
|
5312
|
+
# closed, so the deferred check may legitimately not have run.
|
|
5313
|
+
payload = complete(tasks_md(change_verify=change_verify_c1, evidence=deferred_evidence))
|
|
5314
|
+
if payload.get("status") != "pass":
|
|
5315
|
+
report(f"{label} refused a regression check deferring to a declared change-level check.")
|
|
5316
|
+
report(json.dumps(payload.get("problems", []), indent=2))
|
|
5317
|
+
return 1
|
|
5318
|
+
|
|
5319
|
+
# (b) An untagged check deferring to C1 fails: only a
|
|
5320
|
+
# `(regression)`-tagged check may defer.
|
|
5321
|
+
payload = complete(
|
|
5322
|
+
tasks_md(
|
|
5323
|
+
change_verify=change_verify_c1,
|
|
5324
|
+
commands=(
|
|
5325
|
+
"M1: behavior reaches the public interface",
|
|
5326
|
+
"M2: the full suite stays green",
|
|
5327
|
+
),
|
|
5328
|
+
evidence=(
|
|
5329
|
+
"M1: behavior exercised.",
|
|
5330
|
+
"M1.red: failed before the implementation.",
|
|
5331
|
+
"M1.green: passed after.",
|
|
5332
|
+
"M2: deferred to C1",
|
|
5333
|
+
"M2.red: n/a",
|
|
5334
|
+
"M2.green: n/a",
|
|
5335
|
+
),
|
|
5336
|
+
)
|
|
5337
|
+
)
|
|
5338
|
+
if (
|
|
5339
|
+
payload.get("status") != "fail"
|
|
5340
|
+
or "deferred-evidence-not-regression" not in codes(payload)
|
|
5341
|
+
):
|
|
5342
|
+
report(f"{label} let an untagged check defer to a change-level check.")
|
|
5343
|
+
report(json.dumps(payload, indent=2))
|
|
5344
|
+
return 1
|
|
5345
|
+
|
|
5346
|
+
# (c) A `(regression)`-tagged check deferring to an undeclared C9
|
|
5347
|
+
# fails: the reference must resolve.
|
|
5348
|
+
payload = complete(
|
|
5349
|
+
tasks_md(
|
|
5350
|
+
change_verify=change_verify_c1,
|
|
5351
|
+
evidence=(
|
|
5352
|
+
"M1: behavior exercised.",
|
|
5353
|
+
"M1.red: failed before the implementation.",
|
|
5354
|
+
"M1.green: passed after.",
|
|
5355
|
+
"M2: deferred to C9",
|
|
5356
|
+
),
|
|
5357
|
+
)
|
|
5358
|
+
)
|
|
5359
|
+
if payload.get("status") != "fail" or "deferred-check-unresolved" not in codes(payload):
|
|
5360
|
+
report(f"{label} accepted a deferral to a change-level check that was never declared.")
|
|
5361
|
+
report(json.dumps(payload, indent=2))
|
|
5362
|
+
return 1
|
|
5363
|
+
|
|
5364
|
+
# (d) change-close fails the same way when a task defers but
|
|
5365
|
+
# `## Change Verify` does not exist at all.
|
|
5366
|
+
payload = close(tasks_md(evidence=deferred_evidence))
|
|
5367
|
+
if payload.get("status") != "fail" or "deferred-check-unresolved" not in codes(payload):
|
|
5368
|
+
report(f"{label} closed a change whose deferred check has no Change Verify section.")
|
|
5369
|
+
report(json.dumps(payload, indent=2))
|
|
5370
|
+
return 1
|
|
5371
|
+
|
|
5372
|
+
# (e) change-close fails when C1 is declared but Change Evidence has
|
|
5373
|
+
# no concrete entry for it — the check has to actually run once.
|
|
5374
|
+
payload = close(
|
|
5375
|
+
tasks_md(
|
|
5376
|
+
change_verify=change_verify_c1,
|
|
5377
|
+
change_evidence=change_evidence_c1_pending,
|
|
5378
|
+
evidence=deferred_evidence,
|
|
5379
|
+
)
|
|
5380
|
+
)
|
|
5381
|
+
if payload.get("status") != "fail" or "change-evidence-missing" not in codes(payload):
|
|
5382
|
+
report(f"{label} closed a change whose declared C1 has no concrete Change Evidence.")
|
|
5383
|
+
report(json.dumps(payload, indent=2))
|
|
5384
|
+
return 1
|
|
5385
|
+
|
|
5386
|
+
# (f) change-close passes once Change Evidence carries concrete
|
|
5387
|
+
# evidence for every declared C<n>.
|
|
5388
|
+
payload = close(
|
|
5389
|
+
tasks_md(
|
|
5390
|
+
change_verify=change_verify_c1,
|
|
5391
|
+
change_evidence=change_evidence_c1_pass,
|
|
5392
|
+
evidence=deferred_evidence,
|
|
5393
|
+
)
|
|
5394
|
+
)
|
|
5395
|
+
if payload.get("status") != "pass":
|
|
5396
|
+
report(f"{label} refused a fully resolved deferred change-level check at close.")
|
|
5397
|
+
report(json.dumps(payload.get("problems", []), indent=2))
|
|
5398
|
+
return 1
|
|
5399
|
+
|
|
5400
|
+
# (g) A change no task defers in reports nothing about either
|
|
5401
|
+
# section, whether present or absent — the feature costs nothing to a
|
|
5402
|
+
# change that never uses it, and a declared-but-unused section is
|
|
5403
|
+
# still held to its own shape.
|
|
5404
|
+
undeferred_evidence = (
|
|
5405
|
+
"M1: behavior exercised.",
|
|
5406
|
+
"M1.red: failed before the implementation.",
|
|
5407
|
+
"M1.green: passed after.",
|
|
5408
|
+
"M2: existing suite still green.",
|
|
5409
|
+
)
|
|
5410
|
+
without_sections = close(tasks_md(evidence=undeferred_evidence))
|
|
5411
|
+
with_unused_sections = close(
|
|
5412
|
+
tasks_md(
|
|
5413
|
+
change_verify=change_verify_c1,
|
|
5414
|
+
change_evidence=change_evidence_c1_pass,
|
|
5415
|
+
evidence=undeferred_evidence,
|
|
5416
|
+
)
|
|
5417
|
+
)
|
|
5418
|
+
for payload in (without_sections, with_unused_sections):
|
|
5419
|
+
stray = [
|
|
5420
|
+
item
|
|
5421
|
+
for item in payload.get("problems", [])
|
|
5422
|
+
if item.get("code", "").startswith("change-verify")
|
|
5423
|
+
or item.get("code", "").startswith("change-evidence")
|
|
5424
|
+
or item.get("code", "").startswith("deferred-")
|
|
5425
|
+
]
|
|
5426
|
+
if stray:
|
|
5427
|
+
report(
|
|
5428
|
+
f"{label} reported a Change Verify/Evidence problem for a "
|
|
5429
|
+
"change nothing defers to."
|
|
5430
|
+
)
|
|
5431
|
+
report(json.dumps(stray, indent=2))
|
|
5432
|
+
return 1
|
|
5433
|
+
|
|
5434
|
+
# (h) Non-contiguous C<n> labels fail change-close: the section is
|
|
5435
|
+
# held to the same label discipline `M<n>` already is.
|
|
5436
|
+
broken_change_verify = (
|
|
5437
|
+
"## Change Verify\n\n"
|
|
5438
|
+
"- Strategy: regression-first\n"
|
|
5439
|
+
"- C1: the full three-layer suite is 0 failed\n"
|
|
5440
|
+
"- C3: the byte baseline stays identical\n\n"
|
|
5441
|
+
)
|
|
5442
|
+
payload = close(
|
|
5443
|
+
tasks_md(
|
|
5444
|
+
change_verify=broken_change_verify,
|
|
5445
|
+
change_evidence="## Change Evidence\n\n- C1: pass.\n- C3: pass.\n\n",
|
|
5446
|
+
evidence=deferred_evidence,
|
|
5447
|
+
)
|
|
5448
|
+
)
|
|
5449
|
+
if payload.get("status") != "fail" or "change-verify-shape" not in codes(payload):
|
|
5450
|
+
report(f"{label} accepted non-contiguous Change Verify labels.")
|
|
5451
|
+
report(json.dumps(payload, indent=2))
|
|
5452
|
+
return 1
|
|
5453
|
+
|
|
5454
|
+
report(f"{label} scenario passed.")
|
|
5455
|
+
return 0
|
|
5456
|
+
|
|
5457
|
+
|
|
5458
|
+
def validate_packaged_schema_derivation_scenario() -> int:
|
|
5459
|
+
label = "packaged-schema-derivation"
|
|
5460
|
+
|
|
5461
|
+
# The helper derives the consumer-repo paths every install/uninstall/clear
|
|
5462
|
+
# assertion iterates. When its root stopped existing it returned an empty
|
|
5463
|
+
# list, so those loops compared nothing and reported success. Anchor it to
|
|
5464
|
+
# what the installer really writes, and make emptiness a failure here.
|
|
5465
|
+
try:
|
|
5466
|
+
packaged_openspec_schema_install_paths(ROOT / "no-such-packaged-root")
|
|
5467
|
+
except FileNotFoundError as error:
|
|
5468
|
+
if "no-such-packaged-root" not in str(error):
|
|
5469
|
+
report(f"{label} missing-root failure does not name the path it expected.")
|
|
5470
|
+
report(str(error))
|
|
5471
|
+
return 1
|
|
5472
|
+
else:
|
|
5473
|
+
report(
|
|
5474
|
+
f"{label} returned a set for a missing packaged root instead of failing; "
|
|
5475
|
+
"an absent root must not silently empty its callers' assertions."
|
|
5476
|
+
)
|
|
5477
|
+
return 1
|
|
5478
|
+
|
|
5479
|
+
derived = packaged_openspec_schema_install_paths()
|
|
5480
|
+
if not derived:
|
|
5481
|
+
report(
|
|
5482
|
+
f"{label} derived no packaged schema paths, so every assertion that "
|
|
5483
|
+
"iterates them verifies nothing."
|
|
5484
|
+
)
|
|
5485
|
+
return 1
|
|
5486
|
+
|
|
5487
|
+
with tempfile.TemporaryDirectory(prefix="keel-packaged-schema-") as raw_tmp:
|
|
5488
|
+
repo = Path(raw_tmp) / "repo"
|
|
5489
|
+
repo.mkdir()
|
|
5490
|
+
install = run_keel(repo, "--install")
|
|
5491
|
+
if install.returncode != 0:
|
|
5492
|
+
report(f"{label} keel --install failed.")
|
|
5493
|
+
report((install.stderr or install.stdout).strip())
|
|
5494
|
+
return 1
|
|
5495
|
+
|
|
5496
|
+
schema_root = repo / OPENSPEC_SCHEMA_ROOT
|
|
5497
|
+
installed = sorted(
|
|
5498
|
+
path.relative_to(repo).as_posix()
|
|
5499
|
+
for path in schema_root.rglob("*")
|
|
5500
|
+
if path.is_file()
|
|
5501
|
+
)
|
|
5502
|
+
if installed != sorted(derived):
|
|
5503
|
+
report(
|
|
5504
|
+
f"{label} derived paths do not match what keel --install wrote."
|
|
5505
|
+
)
|
|
5506
|
+
report(f"derived: {sorted(derived)}")
|
|
5507
|
+
report(f"installed: {installed}")
|
|
5508
|
+
return 1
|
|
5509
|
+
|
|
5510
|
+
report(f"{label} scenario passed.")
|
|
5511
|
+
return 0
|
|
5512
|
+
|
|
5513
|
+
|
|
5514
|
+
def validate_invalidation_authoring_surface_scenario() -> int:
|
|
5515
|
+
label = "invalidation-authoring-surface"
|
|
5516
|
+
|
|
5517
|
+
# The two schema copies are the repo-local one OpenSpec resolves and the
|
|
5518
|
+
# packaged one `keel --init` writes. This is the only check that asserts they
|
|
5519
|
+
# agree: compact-task-authoring used to imply it through a projection loop
|
|
5520
|
+
# rooted at trees that no longer exist, so it compared nothing and has since
|
|
5521
|
+
# been removed.
|
|
5522
|
+
for local, packaged in SCHEMA_COPY_PAIRS:
|
|
5523
|
+
local_text = (ROOT / local).read_text(encoding="utf-8")
|
|
5524
|
+
packaged_text = (ROOT / packaged).read_text(encoding="utf-8")
|
|
5525
|
+
if local_text != packaged_text:
|
|
5526
|
+
report(f"{label} schema copies diverge: {local} vs {packaged}")
|
|
5527
|
+
return 1
|
|
5528
|
+
|
|
5529
|
+
template = (
|
|
5530
|
+
ROOT / "openspec/schemas/keel-spec-driven/templates/tasks.md"
|
|
5531
|
+
).read_text(encoding="utf-8")
|
|
5532
|
+
for marker in ("## Invalidates", "- None.", "- I1:"):
|
|
5533
|
+
if marker not in template:
|
|
5534
|
+
report(f"{label} tasks template lacks the invalidation section: {marker}")
|
|
5535
|
+
return 1
|
|
4505
5536
|
|
|
4506
5537
|
schema = (
|
|
4507
5538
|
ROOT / "openspec/schemas/keel-spec-driven/schema.yaml"
|
|
@@ -5124,7 +6155,10 @@ def validate_covers_separator_collision_scenario() -> int:
|
|
|
5124
6155
|
)
|
|
5125
6156
|
report(trimmed_messages)
|
|
5126
6157
|
return 1
|
|
5127
|
-
# A capability with no
|
|
6158
|
+
# A capability with no colliding name receives no separator hint. It
|
|
6159
|
+
# still gets the unresolved diagnostic, which since 5.21.0 names what
|
|
6160
|
+
# the spec holds instead; what is asserted here is only that the
|
|
6161
|
+
# separator explanation stays with the capability that has one.
|
|
5128
6162
|
plain = start(
|
|
5129
6163
|
"plain", "clean-cap / No such requirement / No such scenario"
|
|
5130
6164
|
)
|
|
@@ -5162,6 +6196,195 @@ def validate_covers_separator_collision_scenario() -> int:
|
|
|
5162
6196
|
return 0
|
|
5163
6197
|
|
|
5164
6198
|
|
|
6199
|
+
def validate_unresolved_covers_names_what_failed_scenario() -> int:
|
|
6200
|
+
"""Issue #49, 2026-08-02 supplement: an unresolved reference said only that.
|
|
6201
|
+
|
|
6202
|
+
A two-segment `Covers` reference whose second segment is a Scenario name is
|
|
6203
|
+
the most common way to write the notation wrong — the shipped task template
|
|
6204
|
+
taught it — and the spec being read holds that exact name one heading level
|
|
6205
|
+
down. The refusal reported the reference back and nothing else, while the
|
|
6206
|
+
hierarchy sentence sat thirty lines above in the same function, reachable
|
|
6207
|
+
only by over-segmenting. Every case below still fails; only what it says
|
|
6208
|
+
changes, and the last case asserts that.
|
|
6209
|
+
"""
|
|
6210
|
+
spec = (
|
|
6211
|
+
"# demo-cap\n\n## Purpose\nDemo.\n\n"
|
|
6212
|
+
"### Requirement: The store validates itself\n"
|
|
6213
|
+
"Keel MUST validate the published store.\n\n"
|
|
6214
|
+
"#### Scenario: A published store passes the pinned validator\n"
|
|
6215
|
+
"- **WHEN** the pinned validator runs\n"
|
|
6216
|
+
"- **THEN** the published store passes\n"
|
|
6217
|
+
"- **AND THEN** the pin is reported\n\n"
|
|
6218
|
+
"#### Scenario: A shared scenario name\n"
|
|
6219
|
+
"- **WHEN** a thing\n- **THEN** another\n\n"
|
|
6220
|
+
"### Requirement: Another requirement\n"
|
|
6221
|
+
"Keel MUST do the other thing.\n\n"
|
|
6222
|
+
"#### Scenario: A shared scenario name\n"
|
|
6223
|
+
"- **WHEN** a thing\n- **THEN** another\n"
|
|
6224
|
+
)
|
|
6225
|
+
|
|
6226
|
+
with tempfile.TemporaryDirectory(prefix="keel-covers-what-failed-") as raw:
|
|
6227
|
+
repo = Path(raw)
|
|
6228
|
+
write_text(repo / "openspec/specs/demo-cap/spec.md", spec)
|
|
6229
|
+
|
|
6230
|
+
def start(change: str, reference: str) -> dict:
|
|
6231
|
+
write_text(
|
|
6232
|
+
repo / f"openspec/changes/{change}/tasks.md",
|
|
6233
|
+
task_capsule_compact_fixture().replace(
|
|
6234
|
+
" - E1: Public behavior passes.\n", f" - {reference}\n"
|
|
6235
|
+
),
|
|
6236
|
+
)
|
|
6237
|
+
result = run_keel(
|
|
6238
|
+
repo, "gate", "task-start", "--change", change, "--task", "1.1",
|
|
6239
|
+
"--json",
|
|
6240
|
+
)
|
|
6241
|
+
return json.loads(result.stdout)
|
|
6242
|
+
|
|
6243
|
+
def refusal(change: str, reference: str) -> tuple[dict, str]:
|
|
6244
|
+
payload = start(change, reference)
|
|
6245
|
+
message = " ".join(
|
|
6246
|
+
problem.get("message", "")
|
|
6247
|
+
for problem in payload.get("problems", [])
|
|
6248
|
+
)
|
|
6249
|
+
return payload, message
|
|
6250
|
+
|
|
6251
|
+
# The reported case: the Scenario name written where a Requirement goes.
|
|
6252
|
+
scenario_ref = "demo-cap / A published store passes the pinned validator"
|
|
6253
|
+
offered, offered_message = refusal("offered", scenario_ref)
|
|
6254
|
+
corrected = (
|
|
6255
|
+
"demo-cap / The store validates itself"
|
|
6256
|
+
" / A published store passes the pinned validator"
|
|
6257
|
+
)
|
|
6258
|
+
if offered.get("status") != "fail":
|
|
6259
|
+
report(
|
|
6260
|
+
"unresolved-covers-names-what-failed: a Scenario named as a "
|
|
6261
|
+
"Requirement stopped being refused."
|
|
6262
|
+
)
|
|
6263
|
+
report(json.dumps(offered.get("problems"), ensure_ascii=False))
|
|
6264
|
+
return 1
|
|
6265
|
+
if not any(
|
|
6266
|
+
problem.get("code") == "unresolved-covers"
|
|
6267
|
+
for problem in offered.get("problems", [])
|
|
6268
|
+
):
|
|
6269
|
+
report(
|
|
6270
|
+
"unresolved-covers-names-what-failed: the refusal no longer "
|
|
6271
|
+
"carries the unresolved-covers code."
|
|
6272
|
+
)
|
|
6273
|
+
report(json.dumps(offered.get("problems"), ensure_ascii=False))
|
|
6274
|
+
return 1
|
|
6275
|
+
if (
|
|
6276
|
+
"The store validates itself" not in offered_message
|
|
6277
|
+
or "Scenario" not in offered_message
|
|
6278
|
+
or corrected not in offered_message
|
|
6279
|
+
):
|
|
6280
|
+
report(
|
|
6281
|
+
"unresolved-covers-names-what-failed: the diagnostic did not "
|
|
6282
|
+
"name the Requirement the Scenario belongs to, did not call it "
|
|
6283
|
+
"a Scenario, or did not spell the corrected reference."
|
|
6284
|
+
)
|
|
6285
|
+
report(offered_message)
|
|
6286
|
+
return 1
|
|
6287
|
+
|
|
6288
|
+
# A name the capability declares nowhere: say the spec was read.
|
|
6289
|
+
absent, absent_message = refusal("absent", "demo-cap / No such name")
|
|
6290
|
+
if absent.get("status") != "fail":
|
|
6291
|
+
report(
|
|
6292
|
+
"unresolved-covers-names-what-failed: a name the capability does "
|
|
6293
|
+
"not declare stopped being refused."
|
|
6294
|
+
)
|
|
6295
|
+
report(json.dumps(absent.get("problems"), ensure_ascii=False))
|
|
6296
|
+
return 1
|
|
6297
|
+
if (
|
|
6298
|
+
"No such name" not in absent_message
|
|
6299
|
+
or "hierarchy is capability / requirement" not in absent_message
|
|
6300
|
+
):
|
|
6301
|
+
report(
|
|
6302
|
+
"unresolved-covers-names-what-failed: a name the spec does not "
|
|
6303
|
+
"declare was not reported as read, or the hierarchy was withheld."
|
|
6304
|
+
)
|
|
6305
|
+
report(absent_message)
|
|
6306
|
+
return 1
|
|
6307
|
+
|
|
6308
|
+
# A capability with no spec at all is a different failure.
|
|
6309
|
+
nospec, nospec_message = refusal("nospec", "nosuch-cap / Whatever")
|
|
6310
|
+
if nospec.get("status") != "fail":
|
|
6311
|
+
report(
|
|
6312
|
+
"unresolved-covers-names-what-failed: a capability with no spec "
|
|
6313
|
+
"stopped being refused."
|
|
6314
|
+
)
|
|
6315
|
+
report(json.dumps(nospec.get("problems"), ensure_ascii=False))
|
|
6316
|
+
return 1
|
|
6317
|
+
if (
|
|
6318
|
+
"nosuch-cap" not in nospec_message
|
|
6319
|
+
or "no spec" not in nospec_message.lower()
|
|
6320
|
+
):
|
|
6321
|
+
report(
|
|
6322
|
+
"unresolved-covers-names-what-failed: a capability with no spec "
|
|
6323
|
+
"was not distinguished from a name the spec lacks."
|
|
6324
|
+
)
|
|
6325
|
+
report(nospec_message)
|
|
6326
|
+
return 1
|
|
6327
|
+
|
|
6328
|
+
# Ambiguous: the same Scenario name under two Requirements has no single
|
|
6329
|
+
# correction, so none is offered.
|
|
6330
|
+
shared, shared_message = refusal("shared", "demo-cap / A shared scenario name")
|
|
6331
|
+
if shared.get("status") != "fail":
|
|
6332
|
+
report(
|
|
6333
|
+
"unresolved-covers-names-what-failed: an ambiguous Scenario name "
|
|
6334
|
+
"stopped being refused."
|
|
6335
|
+
)
|
|
6336
|
+
report(json.dumps(shared.get("problems"), ensure_ascii=False))
|
|
6337
|
+
return 1
|
|
6338
|
+
if (
|
|
6339
|
+
"The store validates itself" not in shared_message
|
|
6340
|
+
or "Another requirement" not in shared_message
|
|
6341
|
+
):
|
|
6342
|
+
report(
|
|
6343
|
+
"unresolved-covers-names-what-failed: an ambiguous Scenario name "
|
|
6344
|
+
"did not name the Requirements it appears under."
|
|
6345
|
+
)
|
|
6346
|
+
report(shared_message)
|
|
6347
|
+
return 1
|
|
6348
|
+
if "Write it as" in shared_message:
|
|
6349
|
+
report(
|
|
6350
|
+
"unresolved-covers-names-what-failed: a corrected reference was "
|
|
6351
|
+
"offered for a Scenario name that has more than one parent."
|
|
6352
|
+
)
|
|
6353
|
+
report(shared_message)
|
|
6354
|
+
return 1
|
|
6355
|
+
|
|
6356
|
+
# What resolves is unchanged. This is the assertion that must not be
|
|
6357
|
+
# dropped: reading more of the spec on the failure path is the direction
|
|
6358
|
+
# in which a refusal could accidentally become a resolution.
|
|
6359
|
+
resolves = start("resolves", "demo-cap / The store validates itself")
|
|
6360
|
+
capsule = resolves.get("contract", {}).get("capsule", {})
|
|
6361
|
+
authority = capsule.get("authority", [])
|
|
6362
|
+
if (
|
|
6363
|
+
resolves.get("status") != "pass"
|
|
6364
|
+
or resolves.get("problems")
|
|
6365
|
+
or len(authority) != 1
|
|
6366
|
+
or authority[0].get("kind") != "requirement"
|
|
6367
|
+
or not authority[0]
|
|
6368
|
+
.get("source", "")
|
|
6369
|
+
.endswith("specs/demo-cap/spec.md#Requirement:The store validates itself")
|
|
6370
|
+
):
|
|
6371
|
+
report(
|
|
6372
|
+
"unresolved-covers-names-what-failed: a reference that resolves "
|
|
6373
|
+
"no longer compiles to the same authority."
|
|
6374
|
+
)
|
|
6375
|
+
report(json.dumps(resolves, ensure_ascii=False)[:2000])
|
|
6376
|
+
return 1
|
|
6377
|
+
|
|
6378
|
+
if "unresolved-covers-names-what-failed" not in {name for name, _ in SCENARIOS}:
|
|
6379
|
+
report(
|
|
6380
|
+
"unresolved-covers-names-what-failed: the scenario registry does not "
|
|
6381
|
+
"include it."
|
|
6382
|
+
)
|
|
6383
|
+
return 1
|
|
6384
|
+
report("unresolved-covers-names-what-failed scenario passed.")
|
|
6385
|
+
return 0
|
|
6386
|
+
|
|
6387
|
+
|
|
5165
6388
|
def validate_unresolved_authority_names_field_scenario() -> int:
|
|
5166
6389
|
"""Issue #7 example 3: the diagnostic must name what it actually reads.
|
|
5167
6390
|
|
|
@@ -6216,6 +7439,14 @@ def validate_default_completion_attributes_writes_scenario() -> int:
|
|
|
6216
7439
|
dirty then answers "did this task write it" without asking Git to answer
|
|
6217
7440
|
"which task wrote it", which is the question it cannot answer in a
|
|
6218
7441
|
half-finished change.
|
|
7442
|
+
|
|
7443
|
+
Issue #72: recording only the path, not its content, meant a path already
|
|
7444
|
+
dirty at task start was exempt for the rest of the task's life, even after
|
|
7445
|
+
the task wrote it again — the common case, since most tasks in this
|
|
7446
|
+
repository start on a tree a prior task already legitimately touched. M2
|
|
7447
|
+
proves the fix: a dirty-at-start path the task modifies again is
|
|
7448
|
+
attributed, while M1's own dirty-at-start path (never touched again) stays
|
|
7449
|
+
exempt.
|
|
6219
7450
|
"""
|
|
6220
7451
|
label = "default-completion-attributes-writes"
|
|
6221
7452
|
|
|
@@ -6314,6 +7545,34 @@ def validate_default_completion_attributes_writes_scenario() -> int:
|
|
|
6314
7545
|
)
|
|
6315
7546
|
return 1
|
|
6316
7547
|
|
|
7548
|
+
# M2 — issue #72: a path already dirty at task start is exempt only
|
|
7549
|
+
# while its content stays the one recorded then. A task that goes on
|
|
7550
|
+
# to modify that same path has written it, and the boundary M1 proved
|
|
7551
|
+
# above must still catch that write instead of exempting it forever
|
|
7552
|
+
# because it happened to be dirty at the start too.
|
|
7553
|
+
write_text(
|
|
7554
|
+
repo / "src/already-dirty.js", "// touched again by the task\n"
|
|
7555
|
+
)
|
|
7556
|
+
payload = gate()
|
|
7557
|
+
problems = outside(payload)
|
|
7558
|
+
if not any("src/already-dirty.js" in message for message in problems):
|
|
7559
|
+
report(
|
|
7560
|
+
f"{label} M2 a path already dirty at task start, then "
|
|
7561
|
+
"modified again by the task, was not attributed. Recording "
|
|
7562
|
+
"only the path (not its content) at task start exempts every "
|
|
7563
|
+
"later write to it, not just the one that predates the task."
|
|
7564
|
+
)
|
|
7565
|
+
report(f" status={payload.get('status')!r}")
|
|
7566
|
+
for warning in payload.get("warnings", []):
|
|
7567
|
+
report(f" warning: {warning}")
|
|
7568
|
+
return 1
|
|
7569
|
+
if payload.get("status") != "fail":
|
|
7570
|
+
report(
|
|
7571
|
+
f"{label} M2 the gate named the re-touched path but did not "
|
|
7572
|
+
f"fail; got status {payload.get('status')!r}."
|
|
7573
|
+
)
|
|
7574
|
+
return 1
|
|
7575
|
+
|
|
6317
7576
|
# M1 — an explicit base answers the question the caller asked, which is
|
|
6318
7577
|
# the broader one: everything since that commit, including what was
|
|
6319
7578
|
# already dirty when the task started.
|
|
@@ -7890,23 +9149,214 @@ def validate_non_concrete_check_names_token_scenario() -> int:
|
|
|
7890
9149
|
return 0
|
|
7891
9150
|
|
|
7892
9151
|
|
|
7893
|
-
def
|
|
7894
|
-
"""Issue #
|
|
9152
|
+
def validate_unusable_contract_names_only_its_cause_scenario() -> int:
|
|
9153
|
+
"""Issue #52: the first problem named `Commands`, which compact tasks lack.
|
|
7895
9154
|
|
|
7896
|
-
|
|
7897
|
-
|
|
7898
|
-
|
|
7899
|
-
|
|
9155
|
+
Two defects compound. `missingFieldProblems` emitted a bare "must be
|
|
9156
|
+
concrete" while `unfilledToken` sat eight lines above it, and a contract
|
|
9157
|
+
carrying any diagnostic is discarded, after which `completionChecks` falls
|
|
9158
|
+
back to reading `Commands` — a field a compact task never declares. The
|
|
9159
|
+
derived problem sorts first and is the only one naming a field, so it is the
|
|
9160
|
+
one an author acts on, and it is about a schema they did not choose.
|
|
7900
9161
|
|
|
7901
|
-
|
|
7902
|
-
|
|
9162
|
+
What is deliberately *not* changed is which fields the gate accepts. A bare
|
|
9163
|
+
token in prose stays non-concrete; see the M3 regression on
|
|
9164
|
+
`inline-code-is-concrete` and the decision recorded in
|
|
9165
|
+
keel/archive/follow-ups/2026-07-27-unfilled-token-keywords.md.
|
|
7903
9166
|
"""
|
|
7904
|
-
|
|
7905
|
-
|
|
7906
|
-
|
|
7907
|
-
|
|
7908
|
-
|
|
7909
|
-
|
|
9167
|
+
less, greater = "<", ">"
|
|
9168
|
+
# The reporter's own shape: two bare brackets separated by half a sentence,
|
|
9169
|
+
# so the unbounded `<[^>]+>` swallows the span between them.
|
|
9170
|
+
prose = (
|
|
9171
|
+
f"pass — max ratio 0.001916 (bound {less}0.02), "
|
|
9172
|
+
f"min ratio 0.998107 (bound {greater}0.98)."
|
|
9173
|
+
)
|
|
9174
|
+
captured = f"{less}0.02), min ratio 0.998107 (bound {greater}"
|
|
9175
|
+
review = (
|
|
9176
|
+
" - Review:\n"
|
|
9177
|
+
" - Status: pass\n"
|
|
9178
|
+
" - Acceptance check: reviewed\n"
|
|
9179
|
+
" - Scope check: reviewed\n"
|
|
9180
|
+
" - Findings: none\n"
|
|
9181
|
+
)
|
|
9182
|
+
with tempfile.TemporaryDirectory(prefix="keel-unusable-contract-") as raw:
|
|
9183
|
+
repo = Path(raw)
|
|
9184
|
+
clean = task_capsule_compact_fixture()
|
|
9185
|
+
write_text(repo / "openspec/changes/prose/tasks.md", clean)
|
|
9186
|
+
# The anchor has to be recorded while the Evidence is still concrete —
|
|
9187
|
+
# which is the real sequence, not a workaround. The token arrives when
|
|
9188
|
+
# the author writes up the result, after the task started.
|
|
9189
|
+
if not record_contract_anchor(repo, "prose"):
|
|
9190
|
+
report(
|
|
9191
|
+
"unusable-contract-names-only-its-cause: the clean fixture "
|
|
9192
|
+
"could not record a contract anchor."
|
|
9193
|
+
)
|
|
9194
|
+
return 1
|
|
9195
|
+
started = (repo / "openspec/changes/prose/tasks.md").read_text(
|
|
9196
|
+
encoding="utf-8"
|
|
9197
|
+
)
|
|
9198
|
+
write_text(
|
|
9199
|
+
repo / "openspec/changes/prose/tasks.md",
|
|
9200
|
+
started.replace(
|
|
9201
|
+
" - M1: pending\n", f" - M1: {prose}\n{review}"
|
|
9202
|
+
),
|
|
9203
|
+
)
|
|
9204
|
+
completed = run_keel(
|
|
9205
|
+
repo,
|
|
9206
|
+
"gate",
|
|
9207
|
+
"task-complete",
|
|
9208
|
+
"--change",
|
|
9209
|
+
"prose",
|
|
9210
|
+
"--task",
|
|
9211
|
+
"1.1",
|
|
9212
|
+
"--json",
|
|
9213
|
+
)
|
|
9214
|
+
payload = json.loads(completed.stdout)
|
|
9215
|
+
problems = payload.get("problems", [])
|
|
9216
|
+
messages = [problem.get("message", "") for problem in problems]
|
|
9217
|
+
|
|
9218
|
+
# 1. The derived problem is gone. It named a field compact tasks lack.
|
|
9219
|
+
derived = [text for text in messages if "must define at least one" in text]
|
|
9220
|
+
if derived:
|
|
9221
|
+
report(
|
|
9222
|
+
"unusable-contract-names-only-its-cause: an unusable contract "
|
|
9223
|
+
"still derived a verification-form problem from the other "
|
|
9224
|
+
"schema's field."
|
|
9225
|
+
)
|
|
9226
|
+
for text in derived:
|
|
9227
|
+
report(f" {text}")
|
|
9228
|
+
return 1
|
|
9229
|
+
|
|
9230
|
+
# 2. The remaining problem names the span that caused it, and the escape.
|
|
9231
|
+
named = [
|
|
9232
|
+
text
|
|
9233
|
+
for text in messages
|
|
9234
|
+
if captured in text and "inline code" in text
|
|
9235
|
+
]
|
|
9236
|
+
if not named:
|
|
9237
|
+
report(
|
|
9238
|
+
"unusable-contract-names-only-its-cause: the Evidence "
|
|
9239
|
+
"diagnostic did not name the matched span and the inline-code "
|
|
9240
|
+
"escape."
|
|
9241
|
+
)
|
|
9242
|
+
for text in messages:
|
|
9243
|
+
report(f" {text}")
|
|
9244
|
+
return 1
|
|
9245
|
+
|
|
9246
|
+
# 3. Naming the cause did not stop the gate refusing. This is the
|
|
9247
|
+
# assertion that must never be dropped: suppressing a problem is the
|
|
9248
|
+
# direction that can wrongly make a gate pass.
|
|
9249
|
+
if payload.get("status") != "fail":
|
|
9250
|
+
report(
|
|
9251
|
+
"unusable-contract-names-only-its-cause: the gate returned "
|
|
9252
|
+
f"{payload.get('status')!r} for a task whose Evidence is not "
|
|
9253
|
+
"concrete."
|
|
9254
|
+
)
|
|
9255
|
+
return 1
|
|
9256
|
+
|
|
9257
|
+
# 4. Fencing exactly what the message names clears it, so the offered
|
|
9258
|
+
# repair is the one that works.
|
|
9259
|
+
fenced = started.replace(
|
|
9260
|
+
" - M1: pending\n",
|
|
9261
|
+
f" - M1: `{prose}`\n{review}",
|
|
9262
|
+
)
|
|
9263
|
+
write_text(repo / "openspec/changes/fenced/tasks.md", fenced)
|
|
9264
|
+
if not record_contract_anchor(repo, "fenced"):
|
|
9265
|
+
report(
|
|
9266
|
+
"unusable-contract-names-only-its-cause: fencing the named "
|
|
9267
|
+
"span did not make the Evidence concrete."
|
|
9268
|
+
)
|
|
9269
|
+
return 1
|
|
9270
|
+
|
|
9271
|
+
# 5. An empty field has no token to name, so it keeps the plain wording.
|
|
9272
|
+
empty = clean.replace(
|
|
9273
|
+
" - Covers:\n - E1: Public behavior passes.\n", " - Covers:\n"
|
|
9274
|
+
)
|
|
9275
|
+
write_text(repo / "openspec/changes/empty/tasks.md", empty)
|
|
9276
|
+
bare = run_keel(
|
|
9277
|
+
repo, "gate", "task-start", "--change", "empty", "--task", "1.1",
|
|
9278
|
+
"--json",
|
|
9279
|
+
)
|
|
9280
|
+
bare_messages = [
|
|
9281
|
+
problem.get("message", "")
|
|
9282
|
+
for problem in json.loads(bare.stdout).get("problems", [])
|
|
9283
|
+
]
|
|
9284
|
+
if not any(
|
|
9285
|
+
text.startswith("Covers must be concrete") for text in bare_messages
|
|
9286
|
+
):
|
|
9287
|
+
report(
|
|
9288
|
+
"unusable-contract-names-only-its-cause: an empty required "
|
|
9289
|
+
"field lost the unqualified wording."
|
|
9290
|
+
)
|
|
9291
|
+
for text in bare_messages:
|
|
9292
|
+
report(f" {text}")
|
|
9293
|
+
return 1
|
|
9294
|
+
|
|
9295
|
+
# 6. Suppression must not hide a task that genuinely declares no
|
|
9296
|
+
# verification form. The refusal names the compact field to add.
|
|
9297
|
+
noform = clean.replace(
|
|
9298
|
+
" - Verify:\n - Strategy: evidence-first\n - M1: node test.js\n",
|
|
9299
|
+
"",
|
|
9300
|
+
)
|
|
9301
|
+
write_text(repo / "openspec/changes/noform/tasks.md", noform)
|
|
9302
|
+
absent = run_keel(
|
|
9303
|
+
repo, "gate", "task-complete", "--change", "noform", "--task", "1.1",
|
|
9304
|
+
"--json",
|
|
9305
|
+
)
|
|
9306
|
+
absent_payload = json.loads(absent.stdout)
|
|
9307
|
+
absent_messages = [
|
|
9308
|
+
problem.get("message", "")
|
|
9309
|
+
for problem in absent_payload.get("problems", [])
|
|
9310
|
+
]
|
|
9311
|
+
if absent_payload.get("status") != "fail":
|
|
9312
|
+
report(
|
|
9313
|
+
"unusable-contract-names-only-its-cause: a task declaring no "
|
|
9314
|
+
"verification form was not refused."
|
|
9315
|
+
)
|
|
9316
|
+
return 1
|
|
9317
|
+
if not any("`Verify`" in text for text in absent_messages):
|
|
9318
|
+
report(
|
|
9319
|
+
"unusable-contract-names-only-its-cause: the refusal did not "
|
|
9320
|
+
"name `Verify` as the field to add."
|
|
9321
|
+
)
|
|
9322
|
+
for text in absent_messages:
|
|
9323
|
+
report(f" {text}")
|
|
9324
|
+
return 1
|
|
9325
|
+
if any("must define at least one" in text for text in absent_messages):
|
|
9326
|
+
report(
|
|
9327
|
+
"unusable-contract-names-only-its-cause: a task declaring no "
|
|
9328
|
+
"verification form was told about `Commands`."
|
|
9329
|
+
)
|
|
9330
|
+
return 1
|
|
9331
|
+
if "unusable-contract-names-only-its-cause" not in {
|
|
9332
|
+
name for name, _ in SCENARIOS
|
|
9333
|
+
}:
|
|
9334
|
+
report(
|
|
9335
|
+
"unusable-contract-names-only-its-cause: the scenario registry "
|
|
9336
|
+
"does not include it."
|
|
9337
|
+
)
|
|
9338
|
+
return 1
|
|
9339
|
+
report("unusable-contract-names-only-its-cause scenario passed.")
|
|
9340
|
+
return 0
|
|
9341
|
+
|
|
9342
|
+
|
|
9343
|
+
def validate_covers_question_reference_scope_scenario() -> int:
|
|
9344
|
+
"""Issue #28 item 9: citing a resolved question must not re-open it.
|
|
9345
|
+
|
|
9346
|
+
The question scan used to run over the whole Covers field, so a task that
|
|
9347
|
+
named `Q1` beside the fact that closed it was told to declare a fallback for
|
|
9348
|
+
a question it does not carry. The reporter's only available fix was to
|
|
9349
|
+
delete the reference, which makes traceability worse.
|
|
9350
|
+
|
|
9351
|
+
Both sides are asserted. A scenario that only checked the newly passing
|
|
9352
|
+
shape would also be satisfied by deleting the check outright.
|
|
9353
|
+
"""
|
|
9354
|
+
with tempfile.TemporaryDirectory(prefix="keel-covers-question-") as raw:
|
|
9355
|
+
repo = Path(raw)
|
|
9356
|
+
base = task_capsule_compact_fixture()
|
|
9357
|
+
# Still in scope: the question is the subject of its entry.
|
|
9358
|
+
subject = base.replace(
|
|
9359
|
+
" - E1: Public behavior passes.\n",
|
|
7910
9360
|
" - Q1: Should the widget retry on timeout?\n",
|
|
7911
9361
|
)
|
|
7912
9362
|
write_text(repo / "openspec/changes/subject/tasks.md", subject)
|
|
@@ -8425,12 +9875,13 @@ def validate_findings_resolved_here_scenario() -> int:
|
|
|
8425
9875
|
return 1
|
|
8426
9876
|
|
|
8427
9877
|
# A Findings block normally holds several findings with different
|
|
8428
|
-
# dispositions,
|
|
8429
|
-
#
|
|
8430
|
-
#
|
|
8431
|
-
#
|
|
8432
|
-
#
|
|
8433
|
-
#
|
|
9878
|
+
# dispositions, written as free prose on one line or wrapped across
|
|
9879
|
+
# several — `review-entry-extent` covers the wrapped shape. The
|
|
9880
|
+
# evidence taken from a marker is what follows *that* marker, not the
|
|
9881
|
+
# rest of the text — otherwise the first `Resolved here:` swallows
|
|
9882
|
+
# every disposition after it, and a block mixing a fix with a
|
|
9883
|
+
# tracker-owned follow-up is refused on the follow-up's URL. Measured
|
|
9884
|
+
# on this change's own task 1.3 before it was fixed.
|
|
8434
9885
|
mixed = complete(
|
|
8435
9886
|
"the counter's own line arithmetic was wrong. Resolved here: M1. "
|
|
8436
9887
|
"Second, nothing warned that the CLI was four minors old. "
|
|
@@ -8516,81 +9967,597 @@ def validate_findings_resolved_here_scenario() -> int:
|
|
|
8516
9967
|
return 0
|
|
8517
9968
|
|
|
8518
9969
|
|
|
8519
|
-
|
|
8520
|
-
|
|
9970
|
+
# A task whose Review entries are supplied verbatim, so a fixture can wrap one
|
|
9971
|
+
# across lines. Everything outside the Review block is the complete, checked
|
|
9972
|
+
# shape `tracker_owner_tasks` uses; only the four entries and the sibling below
|
|
9973
|
+
# them vary.
|
|
9974
|
+
def review_extent_tasks(review: str, blocker: str = "none") -> str:
|
|
9975
|
+
return (
|
|
9976
|
+
"# Tasks\n\n## Invalidates\n\n- None.\n\n"
|
|
9977
|
+
"## Expectation Coverage\n\n"
|
|
9978
|
+
"- E1:\n"
|
|
9979
|
+
" - Covered by: 1.1\n\n"
|
|
9980
|
+
"## 1. Work\n\n"
|
|
9981
|
+
"- [x] 1.1 Complete behavior\n"
|
|
9982
|
+
" - Owner: keel-agent\n"
|
|
9983
|
+
" - Mode: implementation\n"
|
|
9984
|
+
" - Covers:\n"
|
|
9985
|
+
" - E1: public behavior\n"
|
|
9986
|
+
" - Read:\n"
|
|
9987
|
+
" - README.md\n"
|
|
9988
|
+
" - Touch:\n"
|
|
9989
|
+
" - src/feature.js\n"
|
|
9990
|
+
" - Commands:\n"
|
|
9991
|
+
" - M1: node test.js\n"
|
|
9992
|
+
" - Acceptance:\n"
|
|
9993
|
+
" - Public behavior passes.\n"
|
|
9994
|
+
" - Autonomy boundary:\n"
|
|
9995
|
+
" - Default: hard-stop\n"
|
|
9996
|
+
" - Pre-authorized fallback: none\n"
|
|
9997
|
+
" - Coupling: none\n"
|
|
9998
|
+
" - Candidate Boundary:\n"
|
|
9999
|
+
" - One candidate.\n"
|
|
10000
|
+
" - Stop Rules:\n"
|
|
10001
|
+
" - Stop on failure.\n"
|
|
10002
|
+
" - Evidence:\n"
|
|
10003
|
+
" - Contract: pending\n"
|
|
10004
|
+
" - M1: passed\n"
|
|
10005
|
+
" - Review:\n"
|
|
10006
|
+
f"{review}"
|
|
10007
|
+
f" - Blocker: {blocker}\n"
|
|
10008
|
+
" - Stop if:\n"
|
|
10009
|
+
" - Scope expands.\n"
|
|
10010
|
+
" - Report:\n"
|
|
10011
|
+
" - Summary\n"
|
|
10012
|
+
)
|
|
8521
10013
|
|
|
8522
|
-
|
|
8523
|
-
|
|
8524
|
-
|
|
10014
|
+
|
|
10015
|
+
def validate_review_entry_extent_scenario() -> int:
|
|
10016
|
+
"""A Review entry is the text the author wrote under its label.
|
|
10017
|
+
|
|
10018
|
+
Issue #49's first supplement. `reviewValue()` matched each Review entry
|
|
10019
|
+
with a line-anchored regex, so an entry that wrapped was judged by its
|
|
10020
|
+
first line and every continuation line was discarded before any check saw
|
|
10021
|
+
it. A `Findings` recording its `Durable owner:` on the fourth line was
|
|
10022
|
+
refused with the owner present, the path existing, and the form correct.
|
|
10023
|
+
|
|
10024
|
+
The same truncation failed open, which #49 does not report: a `Findings`
|
|
10025
|
+
reading `none` on its first line and recording an unowned finding below it
|
|
10026
|
+
produced no problem at all, because the check tested `none` and never saw
|
|
10027
|
+
the rest.
|
|
10028
|
+
|
|
10029
|
+
Both directions are driven through `keel gate task-complete` on real
|
|
10030
|
+
repositories, and the widened read is bounded at the sibling entry so that
|
|
10031
|
+
one entry cannot be satisfied by another's text.
|
|
8525
10032
|
"""
|
|
8526
|
-
|
|
8527
|
-
|
|
8528
|
-
|
|
8529
|
-
|
|
8530
|
-
|
|
8531
|
-
|
|
8532
|
-
|
|
8533
|
-
|
|
8534
|
-
|
|
10033
|
+
label = "review-entry-extent"
|
|
10034
|
+
owner_path = "openspec/changes/demo/tasks.md"
|
|
10035
|
+
|
|
10036
|
+
with tempfile.TemporaryDirectory(
|
|
10037
|
+
prefix="keel-review-extent-", ignore_cleanup_errors=True
|
|
10038
|
+
) as raw:
|
|
10039
|
+
repo = Path(raw)
|
|
10040
|
+
tasks = repo / "openspec/changes/demo/tasks.md"
|
|
10041
|
+
write_text(repo / "openspec/changes/demo/proposal.md", "# Proposal\n")
|
|
10042
|
+
write_text(
|
|
10043
|
+
repo / "openspec/changes/demo/specs/demo/spec.md",
|
|
10044
|
+
"## ADDED Requirements\n",
|
|
10045
|
+
)
|
|
10046
|
+
|
|
10047
|
+
def complete(review: str, blocker: str = "none") -> dict | None:
|
|
10048
|
+
write_text(tasks, review_extent_tasks(review, blocker))
|
|
10049
|
+
if not record_contract_anchor(repo, "demo"):
|
|
10050
|
+
return None
|
|
10051
|
+
result = run_keel(
|
|
10052
|
+
repo, "gate", "task-complete", ".",
|
|
10053
|
+
"--change", "demo", "--task", "1.1", "--json",
|
|
10054
|
+
)
|
|
10055
|
+
return json.loads(result.stdout) if result.stdout.strip() else {}
|
|
10056
|
+
|
|
10057
|
+
def codes(payload: dict) -> list[str]:
|
|
10058
|
+
return sorted(
|
|
10059
|
+
item.get("code", "") for item in payload.get("problems", [])
|
|
10060
|
+
)
|
|
10061
|
+
|
|
10062
|
+
def owner_problems(payload: dict) -> list[str]:
|
|
10063
|
+
return [
|
|
10064
|
+
item.get("message", "")
|
|
10065
|
+
for item in payload.get("problems", [])
|
|
10066
|
+
if item.get("code") == "finding-owner"
|
|
10067
|
+
]
|
|
10068
|
+
|
|
10069
|
+
def readable(payload: dict | None, fixture: str) -> bool:
|
|
10070
|
+
if payload is None:
|
|
10071
|
+
report(
|
|
10072
|
+
f"{label} could not record a Contract anchor for {fixture}: "
|
|
10073
|
+
"task-start refused the fixture, so task-complete never ran "
|
|
10074
|
+
"against it."
|
|
10075
|
+
)
|
|
10076
|
+
return False
|
|
10077
|
+
if not payload:
|
|
10078
|
+
report(
|
|
10079
|
+
f"{label} got no JSON from task-complete for {fixture}, so "
|
|
10080
|
+
"there was no verdict to read rather than a verdict that "
|
|
10081
|
+
"was wrong."
|
|
10082
|
+
)
|
|
10083
|
+
return False
|
|
10084
|
+
return True
|
|
10085
|
+
|
|
10086
|
+
def review_block(findings: str, status: str = "pass") -> str:
|
|
10087
|
+
return (
|
|
10088
|
+
f" - Status: {status}\n"
|
|
10089
|
+
" - Acceptance check: reviewed\n"
|
|
10090
|
+
" - Scope check: reviewed\n"
|
|
10091
|
+
f"{findings}"
|
|
10092
|
+
)
|
|
10093
|
+
|
|
10094
|
+
# M1 — the reported defect. The same finding is written twice: wrapped
|
|
10095
|
+
# across four indented lines with the owner on the last, and joined
|
|
10096
|
+
# onto one line with no word changed. The pair is compared as well as
|
|
10097
|
+
# pinned absolutely, because a comparison alone passes when both forms
|
|
10098
|
+
# are refused identically.
|
|
10099
|
+
wrapped_owner = (
|
|
10100
|
+
" - Findings: two behaviors of state sync turned up in "
|
|
10101
|
+
"measurement.\n"
|
|
10102
|
+
" First, the watcher fires per property rather than once "
|
|
10103
|
+
"per object.\n"
|
|
10104
|
+
" Second, a class-typed property does not update on the "
|
|
10105
|
+
"child side.\n"
|
|
10106
|
+
f" Durable owner: {owner_path}\n"
|
|
10107
|
+
)
|
|
10108
|
+
joined_owner = (
|
|
10109
|
+
" - Findings: two behaviors of state sync turned up in "
|
|
10110
|
+
"measurement. First, the watcher fires per property rather than "
|
|
10111
|
+
"once per object. Second, a class-typed property does not update "
|
|
10112
|
+
f"on the child side. Durable owner: {owner_path}\n"
|
|
10113
|
+
)
|
|
10114
|
+
forms = {
|
|
10115
|
+
"wrapped": complete(review_block(wrapped_owner)),
|
|
10116
|
+
"joined": complete(review_block(joined_owner)),
|
|
10117
|
+
}
|
|
10118
|
+
for form, payload in forms.items():
|
|
10119
|
+
if not readable(payload, f"owner-{form}"):
|
|
10120
|
+
return 1
|
|
10121
|
+
stray = owner_problems(payload)
|
|
10122
|
+
if stray:
|
|
10123
|
+
report(
|
|
10124
|
+
f"{label} refused a durable owner the author recorded, with "
|
|
10125
|
+
f"the finding written {form}. The owner is named after "
|
|
10126
|
+
"`Durable owner:` and the path exists."
|
|
10127
|
+
)
|
|
10128
|
+
report(repr(stray))
|
|
10129
|
+
return 1
|
|
10130
|
+
if codes(forms["wrapped"]) != codes(forms["joined"]):
|
|
8535
10131
|
report(
|
|
8536
|
-
"
|
|
8537
|
-
"
|
|
10132
|
+
f"{label} returned different problems for the same finding "
|
|
10133
|
+
"depending on whether it wrapped."
|
|
8538
10134
|
)
|
|
10135
|
+
report(f"wrapped={codes(forms['wrapped'])} joined={codes(forms['joined'])}")
|
|
8539
10136
|
return 1
|
|
8540
|
-
|
|
8541
|
-
|
|
10137
|
+
|
|
10138
|
+
# M2 — the silent half. `none` on the first line, a real finding under
|
|
10139
|
+
# it. At 5.27.0 this produced no problem at all.
|
|
10140
|
+
below_none = complete(
|
|
10141
|
+
review_block(
|
|
10142
|
+
" - Findings: none\n"
|
|
10143
|
+
" Actually the retry path still drops the last error.\n"
|
|
10144
|
+
" Nobody owns this and no reason is given for dropping "
|
|
10145
|
+
"it.\n"
|
|
10146
|
+
)
|
|
10147
|
+
)
|
|
10148
|
+
if not readable(below_none, "finding-below-none"):
|
|
10149
|
+
return 1
|
|
10150
|
+
if not owner_problems(below_none):
|
|
8542
10151
|
report(
|
|
8543
|
-
"
|
|
8544
|
-
"
|
|
10152
|
+
f"{label} accepted a finding written below a `none` first "
|
|
10153
|
+
"line. The gate read the word `none` and not the finding under "
|
|
10154
|
+
"it."
|
|
8545
10155
|
)
|
|
8546
|
-
report(
|
|
10156
|
+
report(repr(codes(below_none)))
|
|
8547
10157
|
return 1
|
|
8548
10158
|
|
|
8549
|
-
#
|
|
8550
|
-
#
|
|
8551
|
-
#
|
|
8552
|
-
|
|
8553
|
-
|
|
8554
|
-
|
|
8555
|
-
|
|
8556
|
-
return 1
|
|
8557
|
-
write_text(
|
|
8558
|
-
project / "openspec/changes/demo/tasks.md",
|
|
8559
|
-
tracker_owner_tasks("none", "Covered by: 1.1").replace(
|
|
8560
|
-
"- [x] 1.1", "- [ ] 1.1"
|
|
10159
|
+
# M3 — the sibling bound. The wrapped `Findings` carries no disposition
|
|
10160
|
+
# of its own; the `- Blocker:` entry below it does. Reading the entry
|
|
10161
|
+
# whole must not let the sibling's text satisfy it.
|
|
10162
|
+
absorbing = complete(
|
|
10163
|
+
review_block(
|
|
10164
|
+
" - Findings: the helper rewrote its own baseline path.\n"
|
|
10165
|
+
" Nothing here names who owns it.\n"
|
|
8561
10166
|
),
|
|
10167
|
+
blocker=f"none. Durable owner: {owner_path}",
|
|
8562
10168
|
)
|
|
8563
|
-
|
|
8564
|
-
|
|
8565
|
-
|
|
8566
|
-
)
|
|
8567
|
-
if started.returncode != 0 or not (project / "keel/guard.json").is_file():
|
|
10169
|
+
if not readable(absorbing, "sibling-bound"):
|
|
10170
|
+
return 1
|
|
10171
|
+
if not owner_problems(absorbing):
|
|
8568
10172
|
report(
|
|
8569
|
-
"
|
|
8570
|
-
"
|
|
10173
|
+
f"{label} let a `Findings` with no disposition be satisfied by "
|
|
10174
|
+
"the sibling entry below it. A widened read must still stop at "
|
|
10175
|
+
"its sibling."
|
|
8571
10176
|
)
|
|
8572
|
-
report((
|
|
10177
|
+
report(repr(codes(absorbing)))
|
|
8573
10178
|
return 1
|
|
8574
|
-
|
|
8575
|
-
|
|
8576
|
-
|
|
10179
|
+
|
|
10180
|
+
# M4 — the unwrapped shape, which is what all 640 archived Review
|
|
10181
|
+
# entries use. The expected message is written out rather than compared
|
|
10182
|
+
# against another run of the same build, so a change that altered the
|
|
10183
|
+
# text in both places would still fail here.
|
|
10184
|
+
unwrapped = complete(
|
|
10185
|
+
review_block(" - Findings: a finding with no disposition\n")
|
|
8577
10186
|
)
|
|
8578
|
-
if
|
|
10187
|
+
if not readable(unwrapped, "unwrapped"):
|
|
10188
|
+
return 1
|
|
10189
|
+
unwrapped_owner = owner_problems(unwrapped)
|
|
10190
|
+
if len(unwrapped_owner) != 1:
|
|
8579
10191
|
report(
|
|
8580
|
-
"
|
|
8581
|
-
"
|
|
10192
|
+
f"{label} did not produce exactly one `finding-owner` problem "
|
|
10193
|
+
"for an unwrapped finding with no disposition."
|
|
8582
10194
|
)
|
|
8583
|
-
report(
|
|
10195
|
+
report(repr(codes(unwrapped)))
|
|
8584
10196
|
return 1
|
|
8585
|
-
|
|
8586
|
-
|
|
8587
|
-
|
|
8588
|
-
|
|
8589
|
-
|
|
8590
|
-
|
|
10197
|
+
for phrase in (
|
|
10198
|
+
"Review Findings must be `none` or carry a disposition — name a "
|
|
10199
|
+
"path after `Durable owner:`",
|
|
10200
|
+
"Resolved here:",
|
|
10201
|
+
"Durable owner:",
|
|
10202
|
+
"Discard reason:",
|
|
10203
|
+
"keel/HANDOFF.md",
|
|
10204
|
+
):
|
|
10205
|
+
if phrase not in unwrapped_owner[0]:
|
|
10206
|
+
report(
|
|
10207
|
+
f"{label} the unwrapped refusal's message changed; "
|
|
10208
|
+
f"{phrase!r} is no longer in it. This change moves how much "
|
|
10209
|
+
"text is read, not what the gate says."
|
|
10210
|
+
)
|
|
10211
|
+
report(unwrapped_owner[0])
|
|
10212
|
+
return 1
|
|
10213
|
+
passing = complete(
|
|
10214
|
+
review_block(f" - Findings: still open. Durable owner: {owner_path}\n")
|
|
10215
|
+
)
|
|
10216
|
+
if not readable(passing, "unwrapped-owned"):
|
|
10217
|
+
return 1
|
|
10218
|
+
if owner_problems(passing):
|
|
8591
10219
|
report(
|
|
8592
|
-
"
|
|
8593
|
-
"
|
|
10220
|
+
f"{label} refused an unwrapped finding that names an existing "
|
|
10221
|
+
"path after `Durable owner:`."
|
|
10222
|
+
)
|
|
10223
|
+
report(repr(owner_problems(passing)))
|
|
10224
|
+
return 1
|
|
10225
|
+
|
|
10226
|
+
if "review-entry-extent" not in {name for name, _ in SCENARIOS}:
|
|
10227
|
+
report(f"{label}: the scenario registry does not include it.")
|
|
10228
|
+
return 1
|
|
10229
|
+
report(f"{label} scenario passed.")
|
|
10230
|
+
return 0
|
|
10231
|
+
|
|
10232
|
+
|
|
10233
|
+
# A task whose Reauthorizations entry is supplied verbatim, so a fixture can
|
|
10234
|
+
# leave it out entirely, wrap it, or plant an unfilled token on either the
|
|
10235
|
+
# label line or a continuation line. Everything else is the complete, checked
|
|
10236
|
+
# shape `review_extent_tasks` uses; only the Reauthorizations line (or its
|
|
10237
|
+
# absence) and Blocker vary.
|
|
10238
|
+
def reauthorizations_tasks(reauthorizations: str, blocker: str = "none") -> str:
|
|
10239
|
+
return (
|
|
10240
|
+
"# Tasks\n\n## Invalidates\n\n- None.\n\n"
|
|
10241
|
+
"## Expectation Coverage\n\n"
|
|
10242
|
+
"- E1:\n"
|
|
10243
|
+
" - Covered by: 1.1\n\n"
|
|
10244
|
+
"## 1. Work\n\n"
|
|
10245
|
+
"- [x] 1.1 Complete behavior\n"
|
|
10246
|
+
" - Owner: keel-agent\n"
|
|
10247
|
+
" - Mode: implementation\n"
|
|
10248
|
+
" - Covers:\n"
|
|
10249
|
+
" - E1: public behavior\n"
|
|
10250
|
+
" - Read:\n"
|
|
10251
|
+
" - README.md\n"
|
|
10252
|
+
" - Touch:\n"
|
|
10253
|
+
" - src/feature.js\n"
|
|
10254
|
+
" - Commands:\n"
|
|
10255
|
+
" - M1: node test.js\n"
|
|
10256
|
+
" - Acceptance:\n"
|
|
10257
|
+
" - Public behavior passes.\n"
|
|
10258
|
+
" - Autonomy boundary:\n"
|
|
10259
|
+
" - Default: hard-stop\n"
|
|
10260
|
+
" - Pre-authorized fallback: none\n"
|
|
10261
|
+
" - Coupling: none\n"
|
|
10262
|
+
" - Candidate Boundary:\n"
|
|
10263
|
+
" - One candidate.\n"
|
|
10264
|
+
" - Stop Rules:\n"
|
|
10265
|
+
" - Stop on failure.\n"
|
|
10266
|
+
" - Evidence:\n"
|
|
10267
|
+
" - Contract: pending\n"
|
|
10268
|
+
" - M1: passed\n"
|
|
10269
|
+
" - Review:\n"
|
|
10270
|
+
" - Status: pass\n"
|
|
10271
|
+
" - Acceptance check: reviewed\n"
|
|
10272
|
+
" - Scope check: reviewed\n"
|
|
10273
|
+
" - Findings: none\n"
|
|
10274
|
+
f" - Blocker: {blocker}\n"
|
|
10275
|
+
f"{reauthorizations}"
|
|
10276
|
+
" - Stop if:\n"
|
|
10277
|
+
" - Scope expands.\n"
|
|
10278
|
+
" - Report:\n"
|
|
10279
|
+
" - Summary\n"
|
|
10280
|
+
)
|
|
10281
|
+
|
|
10282
|
+
|
|
10283
|
+
def validate_reauthorizations_shape_scenario() -> int:
|
|
10284
|
+
"""A rejection becomes a Reauthorizations entry the author writes (#70).
|
|
10285
|
+
|
|
10286
|
+
The owner chose R2 over R3 on issue #70: the task author records what a
|
|
10287
|
+
gate rejected themselves, in a new `Reauthorizations` Evidence entry; the
|
|
10288
|
+
gate adds no write of its own and validates only the entry's shape. This
|
|
10289
|
+
drives that shape check through `keel gate task-complete` on real
|
|
10290
|
+
repositories: absence and `none` need nothing, a concrete record — wrapped
|
|
10291
|
+
or joined — is accepted, an unfilled `<slot>` token is refused even when it
|
|
10292
|
+
only appears on a continuation line the label line itself does not carry,
|
|
10293
|
+
and a concrete record never blocks completion the way a concrete `Blocker`
|
|
10294
|
+
does.
|
|
10295
|
+
"""
|
|
10296
|
+
label = "reauthorizations-shape"
|
|
10297
|
+
baseline_reauthorizations = " - Reauthorizations: none\n"
|
|
10298
|
+
baseline_blocker = " - Blocker: none\n"
|
|
10299
|
+
|
|
10300
|
+
with tempfile.TemporaryDirectory(
|
|
10301
|
+
prefix="keel-reauthorizations-shape-", ignore_cleanup_errors=True
|
|
10302
|
+
) as raw:
|
|
10303
|
+
repo = Path(raw)
|
|
10304
|
+
tasks = repo / "openspec/changes/demo/tasks.md"
|
|
10305
|
+
write_text(repo / "openspec/changes/demo/proposal.md", "# Proposal\n")
|
|
10306
|
+
write_text(
|
|
10307
|
+
repo / "openspec/changes/demo/specs/demo/spec.md",
|
|
10308
|
+
"## ADDED Requirements\n",
|
|
10309
|
+
)
|
|
10310
|
+
|
|
10311
|
+
# Started once, with clean Evidence, and every variant below edits
|
|
10312
|
+
# only the Reauthorizations/Blocker lines and calls task-complete
|
|
10313
|
+
# directly rather than restarting. Two things make restarting wrong
|
|
10314
|
+
# here: Evidence is not part of the compiled capsule that task-start
|
|
10315
|
+
# fingerprints, so editing only these lines cannot drift it; and
|
|
10316
|
+
# task-start separately enforces its own blanket "Evidence must be
|
|
10317
|
+
# concrete" check across the whole field regardless of which label an
|
|
10318
|
+
# unfilled `<slot>` sits under, so planting one for M1 and then
|
|
10319
|
+
# restarting would be refused there before task-complete's more
|
|
10320
|
+
# specific `reauthorizations-shape` ever saw it.
|
|
10321
|
+
write_text(tasks, reauthorizations_tasks(baseline_reauthorizations))
|
|
10322
|
+
if not record_contract_anchor(repo, "demo"):
|
|
10323
|
+
report(f"{label} could not record a Contract anchor for the baseline fixture.")
|
|
10324
|
+
return 1
|
|
10325
|
+
baseline = tasks.read_text(encoding="utf-8")
|
|
10326
|
+
if baseline_reauthorizations not in baseline or baseline_blocker not in baseline:
|
|
10327
|
+
report(
|
|
10328
|
+
f"{label} lost track of the baseline Reauthorizations/Blocker "
|
|
10329
|
+
"lines after task-start recorded the Contract anchor."
|
|
10330
|
+
)
|
|
10331
|
+
return 1
|
|
10332
|
+
|
|
10333
|
+
def complete(reauthorizations: str, blocker: str = "none") -> dict:
|
|
10334
|
+
text = baseline.replace(baseline_reauthorizations, reauthorizations, 1)
|
|
10335
|
+
if blocker != "none":
|
|
10336
|
+
text = text.replace(baseline_blocker, f" - Blocker: {blocker}\n", 1)
|
|
10337
|
+
write_text(tasks, text)
|
|
10338
|
+
result = run_keel(
|
|
10339
|
+
repo, "gate", "task-complete", ".",
|
|
10340
|
+
"--change", "demo", "--task", "1.1", "--json",
|
|
10341
|
+
)
|
|
10342
|
+
return json.loads(result.stdout) if result.stdout.strip() else {}
|
|
10343
|
+
|
|
10344
|
+
def codes(payload: dict) -> list[str]:
|
|
10345
|
+
return sorted(
|
|
10346
|
+
item.get("code", "") for item in payload.get("problems", [])
|
|
10347
|
+
)
|
|
10348
|
+
|
|
10349
|
+
def shape_problems(payload: dict) -> list[str]:
|
|
10350
|
+
return [
|
|
10351
|
+
item.get("message", "")
|
|
10352
|
+
for item in payload.get("problems", [])
|
|
10353
|
+
if item.get("code") == "reauthorizations-shape"
|
|
10354
|
+
]
|
|
10355
|
+
|
|
10356
|
+
def readable(payload: dict, fixture: str) -> bool:
|
|
10357
|
+
if not payload:
|
|
10358
|
+
report(
|
|
10359
|
+
f"{label} got no JSON from task-complete for {fixture}, so "
|
|
10360
|
+
"there was no verdict to read rather than a verdict that "
|
|
10361
|
+
"was wrong."
|
|
10362
|
+
)
|
|
10363
|
+
return False
|
|
10364
|
+
return True
|
|
10365
|
+
|
|
10366
|
+
# M1 — the core new behavior, in both places an unfilled token can
|
|
10367
|
+
# hide: on the label line itself, and on a continuation line a
|
|
10368
|
+
# first-line-only reader would never see.
|
|
10369
|
+
on_label_line = complete(" - Reauthorizations: <what was rejected>\n")
|
|
10370
|
+
if not readable(on_label_line, "unfilled-on-label-line"):
|
|
10371
|
+
return 1
|
|
10372
|
+
if not shape_problems(on_label_line):
|
|
10373
|
+
report(
|
|
10374
|
+
f"{label} accepted an unfilled `<slot>` token written on the "
|
|
10375
|
+
"Reauthorizations label line itself."
|
|
10376
|
+
)
|
|
10377
|
+
report(repr(codes(on_label_line)))
|
|
10378
|
+
return 1
|
|
10379
|
+
if "<what was rejected>" not in shape_problems(on_label_line)[0]:
|
|
10380
|
+
report(f"{label} refused the unfilled token but did not name it.")
|
|
10381
|
+
report(shape_problems(on_label_line)[0])
|
|
10382
|
+
return 1
|
|
10383
|
+
|
|
10384
|
+
on_continuation = complete(
|
|
10385
|
+
" - Reauthorizations:\n"
|
|
10386
|
+
" - task-complete refused finding-owner on 2026-08-05; added\n"
|
|
10387
|
+
" a Durable owner naming <the tracker issue>.\n"
|
|
10388
|
+
)
|
|
10389
|
+
if not readable(on_continuation, "unfilled-on-continuation-line"):
|
|
10390
|
+
return 1
|
|
10391
|
+
if not shape_problems(on_continuation):
|
|
10392
|
+
report(
|
|
10393
|
+
f"{label} accepted an unfilled `<slot>` token written on a "
|
|
10394
|
+
"continuation line below the Reauthorizations label. A reader "
|
|
10395
|
+
"that only looks at the first line would miss it."
|
|
10396
|
+
)
|
|
10397
|
+
report(repr(codes(on_continuation)))
|
|
10398
|
+
return 1
|
|
10399
|
+
if "<the tracker issue>" not in shape_problems(on_continuation)[0]:
|
|
10400
|
+
report(
|
|
10401
|
+
f"{label} refused the continuation-line token but did not "
|
|
10402
|
+
"name it."
|
|
10403
|
+
)
|
|
10404
|
+
report(shape_problems(on_continuation)[0])
|
|
10405
|
+
return 1
|
|
10406
|
+
|
|
10407
|
+
# M2 (regression) — no false positives: absent, none, and a wrapped
|
|
10408
|
+
# concrete record all pass, and wrapped/joined forms of the same
|
|
10409
|
+
# concrete text agree.
|
|
10410
|
+
absent = complete("")
|
|
10411
|
+
if not readable(absent, "absent"):
|
|
10412
|
+
return 1
|
|
10413
|
+
if shape_problems(absent):
|
|
10414
|
+
report(
|
|
10415
|
+
f"{label} refused a task that declares no Reauthorizations "
|
|
10416
|
+
"entry at all."
|
|
10417
|
+
)
|
|
10418
|
+
report(repr(shape_problems(absent)))
|
|
10419
|
+
return 1
|
|
10420
|
+
|
|
10421
|
+
bare_none = complete(" - Reauthorizations: none\n")
|
|
10422
|
+
if not readable(bare_none, "bare-none"):
|
|
10423
|
+
return 1
|
|
10424
|
+
if shape_problems(bare_none):
|
|
10425
|
+
report(f"{label} refused a bare `Reauthorizations: none`.")
|
|
10426
|
+
report(repr(shape_problems(bare_none)))
|
|
10427
|
+
return 1
|
|
10428
|
+
|
|
10429
|
+
wrapped_concrete = complete(
|
|
10430
|
+
" - Reauthorizations:\n"
|
|
10431
|
+
" - task-complete refused outside-touch on src/helper.js on\n"
|
|
10432
|
+
" 2026-08-05; the file belonged to this task and Touch was\n"
|
|
10433
|
+
" missing it, so it was added.\n"
|
|
10434
|
+
)
|
|
10435
|
+
joined_concrete = complete(
|
|
10436
|
+
" - Reauthorizations: task-complete refused outside-touch on "
|
|
10437
|
+
"src/helper.js on 2026-08-05; the file belonged to this task and "
|
|
10438
|
+
"Touch was missing it, so it was added.\n"
|
|
10439
|
+
)
|
|
10440
|
+
if not readable(wrapped_concrete, "wrapped-concrete"):
|
|
10441
|
+
return 1
|
|
10442
|
+
if not readable(joined_concrete, "joined-concrete"):
|
|
10443
|
+
return 1
|
|
10444
|
+
if shape_problems(wrapped_concrete):
|
|
10445
|
+
report(f"{label} refused a concrete, wrapped Reauthorizations record.")
|
|
10446
|
+
report(repr(shape_problems(wrapped_concrete)))
|
|
10447
|
+
return 1
|
|
10448
|
+
if codes(wrapped_concrete) != codes(joined_concrete):
|
|
10449
|
+
report(
|
|
10450
|
+
f"{label} returned different problems for the same concrete "
|
|
10451
|
+
"record depending on whether it wrapped."
|
|
10452
|
+
)
|
|
10453
|
+
report(
|
|
10454
|
+
f"wrapped={codes(wrapped_concrete)} joined={codes(joined_concrete)}"
|
|
10455
|
+
)
|
|
10456
|
+
return 1
|
|
10457
|
+
|
|
10458
|
+
# M3 (regression) — presence does not block completion, unlike
|
|
10459
|
+
# Blocker. Proven able to fail: a concrete Blocker on the same
|
|
10460
|
+
# repository does fail it, so the comparison is not vacuous.
|
|
10461
|
+
if codes(wrapped_concrete):
|
|
10462
|
+
report(
|
|
10463
|
+
f"{label} a concrete Reauthorizations record produced a "
|
|
10464
|
+
"problem — presence alone must not fail completion."
|
|
10465
|
+
)
|
|
10466
|
+
report(repr(codes(wrapped_concrete)))
|
|
10467
|
+
return 1
|
|
10468
|
+
concrete_blocker = complete("", blocker="a real blocker is recorded here")
|
|
10469
|
+
if not readable(concrete_blocker, "concrete-blocker"):
|
|
10470
|
+
return 1
|
|
10471
|
+
if "blocker" not in codes(concrete_blocker):
|
|
10472
|
+
report(
|
|
10473
|
+
f"{label} the comparison fixture proves nothing: a concrete "
|
|
10474
|
+
"Blocker on this repository did not fail completion either."
|
|
10475
|
+
)
|
|
10476
|
+
report(repr(codes(concrete_blocker)))
|
|
10477
|
+
return 1
|
|
10478
|
+
|
|
10479
|
+
if label not in {name for name, _ in SCENARIOS}:
|
|
10480
|
+
report(f"{label}: the scenario registry does not include it.")
|
|
10481
|
+
return 1
|
|
10482
|
+
report(f"{label} scenario passed.")
|
|
10483
|
+
return 0
|
|
10484
|
+
|
|
10485
|
+
|
|
10486
|
+
def validate_guard_manifest_ignored_scenario() -> int:
|
|
10487
|
+
"""Issue #11: the guard manifest was written but declared ignorable nowhere.
|
|
10488
|
+
|
|
10489
|
+
Every gate run left an untracked `keel/guard.json` in the project, and
|
|
10490
|
+
because completion attributes working-tree paths against Touch, that is a
|
|
10491
|
+
permanent exception the author re-adjudicates at every completion.
|
|
10492
|
+
"""
|
|
10493
|
+
with tempfile.TemporaryDirectory(prefix="keel-guard-ignore-") as raw:
|
|
10494
|
+
project = Path(raw)
|
|
10495
|
+
init = run_keel(project, "--install", "--target", "claude")
|
|
10496
|
+
if init.returncode != 0:
|
|
10497
|
+
report("guard-manifest-ignored: keel --install failed.")
|
|
10498
|
+
report((init.stderr or init.stdout).strip())
|
|
10499
|
+
return 1
|
|
10500
|
+
ignore_path = project / "keel/.gitignore"
|
|
10501
|
+
if not ignore_path.is_file():
|
|
10502
|
+
report(
|
|
10503
|
+
"guard-manifest-ignored: keel --install did not scaffold "
|
|
10504
|
+
"keel/.gitignore."
|
|
10505
|
+
)
|
|
10506
|
+
return 1
|
|
10507
|
+
declared = ignore_path.read_text(encoding="utf-8")
|
|
10508
|
+
if "guard.json" not in declared:
|
|
10509
|
+
report(
|
|
10510
|
+
"guard-manifest-ignored: the scaffolded keel/.gitignore does "
|
|
10511
|
+
"not declare the guard manifest."
|
|
10512
|
+
)
|
|
10513
|
+
report(declared)
|
|
10514
|
+
return 1
|
|
10515
|
+
|
|
10516
|
+
# git must actually honour the declaration: initialize a repository,
|
|
10517
|
+
# write a manifest through a passing task-start, and confirm the path
|
|
10518
|
+
# never appears in porcelain status.
|
|
10519
|
+
if subprocess.run(
|
|
10520
|
+
["git", "init", "--quiet"], cwd=project, capture_output=True
|
|
10521
|
+
).returncode != 0:
|
|
10522
|
+
report("guard-manifest-ignored: git init failed in the fixture.")
|
|
10523
|
+
return 1
|
|
10524
|
+
write_text(
|
|
10525
|
+
project / "openspec/changes/demo/tasks.md",
|
|
10526
|
+
tracker_owner_tasks("none", "Covered by: 1.1").replace(
|
|
10527
|
+
"- [x] 1.1", "- [ ] 1.1"
|
|
10528
|
+
),
|
|
10529
|
+
)
|
|
10530
|
+
started = run_keel(
|
|
10531
|
+
project, "gate", "task-start",
|
|
10532
|
+
"--change", "demo", "--task", "1.1", "--json",
|
|
10533
|
+
)
|
|
10534
|
+
if started.returncode != 0 or not (project / "keel/guard.json").is_file():
|
|
10535
|
+
report(
|
|
10536
|
+
"guard-manifest-ignored: the fixture did not produce a guard "
|
|
10537
|
+
"manifest to test the declaration against."
|
|
10538
|
+
)
|
|
10539
|
+
report((started.stderr or started.stdout).strip())
|
|
10540
|
+
return 1
|
|
10541
|
+
status = subprocess.run(
|
|
10542
|
+
["git", "status", "--short", "--untracked-files=all"],
|
|
10543
|
+
cwd=project, capture_output=True, encoding="utf-8",
|
|
10544
|
+
)
|
|
10545
|
+
if "guard.json" in (status.stdout or ""):
|
|
10546
|
+
report(
|
|
10547
|
+
"guard-manifest-ignored: git still reports the guard manifest "
|
|
10548
|
+
"after a gate run, so the declaration does not take effect."
|
|
10549
|
+
)
|
|
10550
|
+
report(status.stdout or "")
|
|
10551
|
+
return 1
|
|
10552
|
+
|
|
10553
|
+
# Scaffold once: a project's own file is never rewritten.
|
|
10554
|
+
own = "# mine\nguard.json\nscratch/\n"
|
|
10555
|
+
write_text(ignore_path, own)
|
|
10556
|
+
again = run_keel(project, "--install", "--target", "claude")
|
|
10557
|
+
if again.returncode != 0 or ignore_path.read_text(encoding="utf-8") != own:
|
|
10558
|
+
report(
|
|
10559
|
+
"guard-manifest-ignored: a second install overwrote the "
|
|
10560
|
+
"project's own keel/.gitignore."
|
|
8594
10561
|
)
|
|
8595
10562
|
report((again.stderr or again.stdout).strip())
|
|
8596
10563
|
return 1
|
|
@@ -8709,6 +10676,38 @@ def validate_guard_status_is_not_enforcement_scenario() -> int:
|
|
|
8709
10676
|
return 0
|
|
8710
10677
|
|
|
8711
10678
|
|
|
10679
|
+
def validate_guard_warnings_are_concise_scenario() -> int:
|
|
10680
|
+
"""Issue #92 item 1: shorten the two standing guard warnings.
|
|
10681
|
+
|
|
10682
|
+
The owner authorized shortening wording, not removing content: every
|
|
10683
|
+
idea `guard-status-is-not-enforcement` checks for must still resolve, but
|
|
10684
|
+
the combined text must drop below the byte counts `#92` measured before
|
|
10685
|
+
this fix (398 for `guard status`, 397 for `guard clear`, on a fresh
|
|
10686
|
+
directory with no manifest).
|
|
10687
|
+
"""
|
|
10688
|
+
baseline = {"status": 398, "clear": 397}
|
|
10689
|
+
with tempfile.TemporaryDirectory(prefix="keel-guard-concise-") as raw:
|
|
10690
|
+
project = Path(raw)
|
|
10691
|
+
for label, before in baseline.items():
|
|
10692
|
+
result = run_keel(project, "guard", label)
|
|
10693
|
+
out = result.stdout or ""
|
|
10694
|
+
if len(out) >= before:
|
|
10695
|
+
report(
|
|
10696
|
+
f"guard-warnings-are-concise: keel guard {label} is "
|
|
10697
|
+
f"{len(out)} chars, not shorter than the {before}-char "
|
|
10698
|
+
"baseline #92 measured before the wording was shortened."
|
|
10699
|
+
)
|
|
10700
|
+
return 1
|
|
10701
|
+
if "guard-warnings-are-concise" not in {name for name, _ in SCENARIOS}:
|
|
10702
|
+
report(
|
|
10703
|
+
"guard-warnings-are-concise: the scenario registry does not "
|
|
10704
|
+
"include it."
|
|
10705
|
+
)
|
|
10706
|
+
return 1
|
|
10707
|
+
report("guard-warnings-are-concise scenario passed.")
|
|
10708
|
+
return 0
|
|
10709
|
+
|
|
10710
|
+
|
|
8712
10711
|
def validate_source_repo_cli_resolution_scenario() -> int:
|
|
8713
10712
|
"""Issue #13 item 3: a bare `keel` runs the installed package.
|
|
8714
10713
|
|
|
@@ -9648,6 +11647,38 @@ def validate_core_gates_scenario() -> int:
|
|
|
9648
11647
|
report((handoff_owner.stderr or handoff_owner.stdout).strip())
|
|
9649
11648
|
return 1
|
|
9650
11649
|
|
|
11650
|
+
# The accepted-forms enumeration is a menu; the one actionable
|
|
11651
|
+
# instruction inside it (name a path after `Durable owner:`) must lead
|
|
11652
|
+
# the message, not close it, so a reader who forgot to name a path
|
|
11653
|
+
# reads what to do before reading the menu. Three single-cause checks,
|
|
11654
|
+
# not one compound condition — issue #43 exists to catch exactly that
|
|
11655
|
+
# shape.
|
|
11656
|
+
finding_owner_lower = finding_owner_message.lower()
|
|
11657
|
+
instruction_at = finding_owner_lower.find(
|
|
11658
|
+
"name a path after `durable owner:`"
|
|
11659
|
+
)
|
|
11660
|
+
if instruction_at < 0:
|
|
11661
|
+
report(
|
|
11662
|
+
"core-gates scenario: the finding-owner message dropped its "
|
|
11663
|
+
"actionable instruction (\"name a path after `Durable "
|
|
11664
|
+
"owner:`\")."
|
|
11665
|
+
)
|
|
11666
|
+
return 1
|
|
11667
|
+
resolved_here_at = finding_owner_lower.find("`resolved here:`")
|
|
11668
|
+
if resolved_here_at < 0:
|
|
11669
|
+
report(
|
|
11670
|
+
"core-gates scenario: the finding-owner message dropped its "
|
|
11671
|
+
"`Resolved here:` disposition form."
|
|
11672
|
+
)
|
|
11673
|
+
return 1
|
|
11674
|
+
if instruction_at > resolved_here_at:
|
|
11675
|
+
report(
|
|
11676
|
+
"core-gates scenario: the finding-owner message states its "
|
|
11677
|
+
"actionable instruction after the `Resolved here:` form "
|
|
11678
|
+
"instead of before it."
|
|
11679
|
+
)
|
|
11680
|
+
return 1
|
|
11681
|
+
|
|
9651
11682
|
write_text(
|
|
9652
11683
|
completion_tasks,
|
|
9653
11684
|
completion_task(
|
|
@@ -13644,28 +15675,121 @@ def validate_standing_authorization_declaration_scenario() -> int:
|
|
|
13644
15675
|
return 0
|
|
13645
15676
|
|
|
13646
15677
|
|
|
13647
|
-
def
|
|
13648
|
-
|
|
13649
|
-
|
|
13650
|
-
|
|
13651
|
-
|
|
13652
|
-
|
|
13653
|
-
|
|
13654
|
-
|
|
13655
|
-
|
|
13656
|
-
|
|
13657
|
-
|
|
13658
|
-
|
|
13659
|
-
|
|
13660
|
-
|
|
13661
|
-
|
|
13662
|
-
|
|
13663
|
-
|
|
13664
|
-
"
|
|
13665
|
-
|
|
13666
|
-
|
|
13667
|
-
|
|
13668
|
-
|
|
15678
|
+
def validate_standing_authorization_sync_confusion_scenario() -> int:
|
|
15679
|
+
"""`sync` is a `change-close --action` value, not an `authorize:` name (#93).
|
|
15680
|
+
|
|
15681
|
+
A reader who copies `sync` from `change-close --action sync|archive` into
|
|
15682
|
+
`authorize:` gets the generic unrecognized-action message, which never says
|
|
15683
|
+
why `sync` in particular is wrong or what to write instead — and the
|
|
15684
|
+
failure was reported only by `keel --doctor`, an explicitly-invoked
|
|
15685
|
+
diagnostic, not by `keel context`, the command a session runs first.
|
|
15686
|
+
"""
|
|
15687
|
+
label = "standing-authorization-sync-confusion"
|
|
15688
|
+
with tempfile.TemporaryDirectory(prefix="keel-authorize-sync-") as raw_tmp:
|
|
15689
|
+
root = Path(raw_tmp)
|
|
15690
|
+
|
|
15691
|
+
# M1 — `sync` gets a sentence naming the `change-close --action`
|
|
15692
|
+
# confusion and pointing at `archive`; an unrelated typo does not.
|
|
15693
|
+
sync_repo = root / "sync"
|
|
15694
|
+
sync_repo.mkdir()
|
|
15695
|
+
write_authorize_config(sync_repo, "authorize:\n - sync\n")
|
|
15696
|
+
doctor = run_keel(sync_repo, "--doctor")
|
|
15697
|
+
combined = doctor.stdout + doctor.stderr
|
|
15698
|
+
for needle in ("change-close --action", "declare `archive`"):
|
|
15699
|
+
if needle not in combined:
|
|
15700
|
+
report(
|
|
15701
|
+
f"{label} M1 the doctor message does not name the sync "
|
|
15702
|
+
f"confusion: {needle!r}."
|
|
15703
|
+
)
|
|
15704
|
+
report(combined)
|
|
15705
|
+
return 1
|
|
15706
|
+
|
|
15707
|
+
typo_repo = root / "typo"
|
|
15708
|
+
typo_repo.mkdir()
|
|
15709
|
+
write_authorize_config(typo_repo, "authorize:\n - deploy\n")
|
|
15710
|
+
typo_doctor = run_keel(typo_repo, "--doctor")
|
|
15711
|
+
typo_combined = typo_doctor.stdout + typo_doctor.stderr
|
|
15712
|
+
if "deploy" not in typo_combined:
|
|
15713
|
+
report(f"{label} M1 the typo repo's own offending entry went unreported.")
|
|
15714
|
+
report(typo_combined)
|
|
15715
|
+
return 1
|
|
15716
|
+
if "change-close --action" in typo_combined:
|
|
15717
|
+
report(
|
|
15718
|
+
f"{label} M1 a genuine typo (`deploy`) still received the "
|
|
15719
|
+
"sync-specific sentence."
|
|
15720
|
+
)
|
|
15721
|
+
report(typo_combined)
|
|
15722
|
+
return 1
|
|
15723
|
+
|
|
15724
|
+
# M2 — the same broken declaration surfaces in `keel context`'s
|
|
15725
|
+
# warnings, and does not disturb status or nextAction, exactly as an
|
|
15726
|
+
# uncommitted git path is reported without changing selection.
|
|
15727
|
+
context = json.loads(run_keel(sync_repo, "context", "--json").stdout)
|
|
15728
|
+
warnings = " ".join(context.get("warnings", []))
|
|
15729
|
+
if "change-close --action" not in warnings or "sync" not in warnings:
|
|
15730
|
+
report(
|
|
15731
|
+
f"{label} M2 `keel context` did not report the broken "
|
|
15732
|
+
"authorize: declaration."
|
|
15733
|
+
)
|
|
15734
|
+
report(f" warnings: {context.get('warnings')!r}")
|
|
15735
|
+
return 1
|
|
15736
|
+
if context.get("status") != "idle":
|
|
15737
|
+
report(f"{label} M2 the broken declaration changed context's status: {context.get('status')!r}")
|
|
15738
|
+
return 1
|
|
15739
|
+
if context.get("nextAction", {}).get("kind") != "none":
|
|
15740
|
+
report(
|
|
15741
|
+
"{} M2 the broken declaration changed context's nextAction: {!r}".format(
|
|
15742
|
+
label, context.get("nextAction")
|
|
15743
|
+
)
|
|
15744
|
+
)
|
|
15745
|
+
return 1
|
|
15746
|
+
|
|
15747
|
+
typo_context = json.loads(run_keel(typo_repo, "context", "--json").stdout)
|
|
15748
|
+
typo_warnings = " ".join(typo_context.get("warnings", []))
|
|
15749
|
+
if "deploy" not in typo_warnings:
|
|
15750
|
+
report(
|
|
15751
|
+
f"{label} M2 `keel context` did not report the typo repo's "
|
|
15752
|
+
"broken authorize: declaration at all."
|
|
15753
|
+
)
|
|
15754
|
+
report(f" warnings: {typo_context.get('warnings')!r}")
|
|
15755
|
+
return 1
|
|
15756
|
+
if "change-close --action" in typo_warnings:
|
|
15757
|
+
report(
|
|
15758
|
+
f"{label} M2 a genuine typo (`deploy`) still received the "
|
|
15759
|
+
"sync-specific sentence inside `keel context`'s warning."
|
|
15760
|
+
)
|
|
15761
|
+
report(f" warnings: {typo_context.get('warnings')!r}")
|
|
15762
|
+
return 1
|
|
15763
|
+
|
|
15764
|
+
if label not in {name for name, _ in SCENARIOS}:
|
|
15765
|
+
report(f"{label}: the scenario registry does not include it.")
|
|
15766
|
+
return 1
|
|
15767
|
+
report(f"{label} scenario passed.")
|
|
15768
|
+
return 0
|
|
15769
|
+
|
|
15770
|
+
|
|
15771
|
+
def standing_authorization_task(boundary: str = "") -> str:
|
|
15772
|
+
return (
|
|
15773
|
+
"- [ ] 1.1 Behavior\n"
|
|
15774
|
+
" - Covers:\n"
|
|
15775
|
+
" - E1: public behavior\n"
|
|
15776
|
+
" - Touch:\n"
|
|
15777
|
+
" - src/feature.js\n"
|
|
15778
|
+
" - Verify:\n"
|
|
15779
|
+
" - Strategy: evidence-first\n"
|
|
15780
|
+
" - M1: node test.js proves the public behavior\n"
|
|
15781
|
+
+ boundary
|
|
15782
|
+
+ " - Evidence:\n"
|
|
15783
|
+
" - Contract: pending\n"
|
|
15784
|
+
" - M1: pending\n"
|
|
15785
|
+
" - Review:\n"
|
|
15786
|
+
" - Status: pending\n"
|
|
15787
|
+
" - Acceptance check: pending\n"
|
|
15788
|
+
" - Scope check: pending\n"
|
|
15789
|
+
" - Findings: pending\n"
|
|
15790
|
+
" - Blocker: none\n"
|
|
15791
|
+
)
|
|
15792
|
+
|
|
13669
15793
|
|
|
13670
15794
|
def standing_authorization_autonomy(repo: Path) -> list[str] | None:
|
|
13671
15795
|
result = run_keel(
|
|
@@ -14050,6 +16174,273 @@ def validate_triage_declaration_scenario() -> int:
|
|
|
14050
16174
|
return 0
|
|
14051
16175
|
|
|
14052
16176
|
|
|
16177
|
+
def validate_triage_admits_from_the_repository_scenario() -> int:
|
|
16178
|
+
"""The owner's decision belongs in the owner's file, not on the reporter's issue.
|
|
16179
|
+
|
|
16180
|
+
A label records "this may run unattended" in a field the person who filed
|
|
16181
|
+
the issue can see, in the vocabulary they were asked to file under. The
|
|
16182
|
+
second source moves that decision into keel/config.yaml without giving it
|
|
16183
|
+
up to inference: a number is applied by hand to one issue, exactly as a
|
|
16184
|
+
label is. Reported as #62.
|
|
16185
|
+
"""
|
|
16186
|
+
|
|
16187
|
+
def declare(repo: Path, body: str) -> None:
|
|
16188
|
+
(repo / "keel").mkdir(parents=True, exist_ok=True)
|
|
16189
|
+
(repo / "keel" / "config.yaml").write_text(
|
|
16190
|
+
f"fast_check: echo check\n{body}", encoding="utf-8"
|
|
16191
|
+
)
|
|
16192
|
+
|
|
16193
|
+
def triage(repo: Path, *args: str) -> dict | None:
|
|
16194
|
+
result = run_keel(repo, "triage", ".", *args, "--json")
|
|
16195
|
+
try:
|
|
16196
|
+
return json.loads(result.stdout)
|
|
16197
|
+
except json.JSONDecodeError:
|
|
16198
|
+
# Reported here rather than at the call site. "did not admit" and
|
|
16199
|
+
# "did not run" are two different diagnoses, and a caller checking
|
|
16200
|
+
# `verdict is None or verdict["status"] != "admit"` would send the
|
|
16201
|
+
# reader looking for a verdict in a run that produced none.
|
|
16202
|
+
report(
|
|
16203
|
+
f"keel triage {' '.join(args)} printed no JSON "
|
|
16204
|
+
f"(exit {result.returncode}): "
|
|
16205
|
+
f"{(result.stderr or result.stdout).strip()}"
|
|
16206
|
+
)
|
|
16207
|
+
return None
|
|
16208
|
+
|
|
16209
|
+
def make(root: Path, name: str, body: str) -> Path:
|
|
16210
|
+
repo = root / name
|
|
16211
|
+
repo.mkdir()
|
|
16212
|
+
declare(repo, body)
|
|
16213
|
+
return repo
|
|
16214
|
+
|
|
16215
|
+
nested = "triage:\n labels:\n - auto\n issues:\n - 62\n"
|
|
16216
|
+
|
|
16217
|
+
with tempfile.TemporaryDirectory(prefix="keel-triage-repo-") as raw_tmp:
|
|
16218
|
+
root = Path(raw_tmp)
|
|
16219
|
+
|
|
16220
|
+
# M1 — a number listed in the repository's own file admits, and says so.
|
|
16221
|
+
numbers = make(root, "numbers", "triage:\n issues:\n - 62\n - 58\n")
|
|
16222
|
+
admitted = triage(numbers, "--issue", "62", "--labels", "bug")
|
|
16223
|
+
if admitted is None or admitted.get("status") != "admit":
|
|
16224
|
+
report(f"M1 a declared issue number did not admit: {admitted}")
|
|
16225
|
+
return 1
|
|
16226
|
+
reason = admitted.get("reason") or ""
|
|
16227
|
+
if "62" not in reason or "keel/config.yaml" not in reason:
|
|
16228
|
+
report(
|
|
16229
|
+
"M1 the admission does not name the number and the file that "
|
|
16230
|
+
f"lists it, so a reader cannot tell which source answered: {reason}"
|
|
16231
|
+
)
|
|
16232
|
+
return 1
|
|
16233
|
+
if admitted.get("sources") != ["issue"]:
|
|
16234
|
+
report(f"M1 the admitting source is not reported as the issue: {admitted}")
|
|
16235
|
+
return 1
|
|
16236
|
+
refused = triage(numbers, "--issue", "61", "--labels", "bug")
|
|
16237
|
+
if refused is None or refused.get("status") != "refuse":
|
|
16238
|
+
report(f"M1 an unlisted number was admitted: {refused}")
|
|
16239
|
+
return 1
|
|
16240
|
+
|
|
16241
|
+
# M2 — the flat form is what every installed repository has. Its
|
|
16242
|
+
# entries keep their exact meaning, down to the reason text, and none
|
|
16243
|
+
# of them is read as a number.
|
|
16244
|
+
flat = make(root, "flat", "triage:\n - auto\n")
|
|
16245
|
+
legacy_admit = triage(flat, "--labels", "auto,bug")
|
|
16246
|
+
if legacy_admit is None or legacy_admit.get("status") != "admit":
|
|
16247
|
+
report(f"M2 the bare list stopped admitting its labels: {legacy_admit}")
|
|
16248
|
+
return 1
|
|
16249
|
+
expected_admit = (
|
|
16250
|
+
"admitted by declared label auto; admission starts work and decides "
|
|
16251
|
+
"nothing after it — every later gate still applies and a material "
|
|
16252
|
+
"decision still stops for the owner."
|
|
16253
|
+
)
|
|
16254
|
+
if (legacy_admit.get("reason") or "") != expected_admit:
|
|
16255
|
+
report(
|
|
16256
|
+
"M2 the bare list's admission reason changed from the 5.23.0 "
|
|
16257
|
+
f"text: {legacy_admit.get('reason')!r}"
|
|
16258
|
+
)
|
|
16259
|
+
return 1
|
|
16260
|
+
if legacy_admit.get("sources") != ["label"]:
|
|
16261
|
+
report(f"M2 the bare list's admitting source is not the label: {legacy_admit}")
|
|
16262
|
+
return 1
|
|
16263
|
+
legacy_refuse = triage(flat, "--labels", "bug")
|
|
16264
|
+
expected_refuse = "the issue carries bug and this repository accepts auto."
|
|
16265
|
+
if legacy_refuse is None or legacy_refuse.get("status") != "refuse":
|
|
16266
|
+
report(f"M2 the bare list admitted an undeclared label: {legacy_refuse}")
|
|
16267
|
+
return 1
|
|
16268
|
+
if (legacy_refuse.get("reason") or "") != expected_refuse:
|
|
16269
|
+
report(
|
|
16270
|
+
"M2 the bare list's refusal reason changed from the 5.23.0 "
|
|
16271
|
+
f"text: {legacy_refuse.get('reason')!r}"
|
|
16272
|
+
)
|
|
16273
|
+
return 1
|
|
16274
|
+
# The entry is the label `auto`, not the number 62 — a bare token must
|
|
16275
|
+
# not have been reclassified, because that would move an authorization
|
|
16276
|
+
# boundary in a repository nobody edited.
|
|
16277
|
+
by_number = triage(flat, "--issue", "62", "--labels", "bug")
|
|
16278
|
+
if by_number is None or by_number.get("status") != "refuse":
|
|
16279
|
+
report(f"M2 a bare list entry was read as an issue number: {by_number}")
|
|
16280
|
+
return 1
|
|
16281
|
+
|
|
16282
|
+
# M2 — and either source admits alone when both are declared.
|
|
16283
|
+
both = make(root, "both", nested)
|
|
16284
|
+
for args, source in (
|
|
16285
|
+
(("--labels", "auto"), "label"),
|
|
16286
|
+
(("--issue", "62"), "issue"),
|
|
16287
|
+
):
|
|
16288
|
+
verdict = triage(both, *args)
|
|
16289
|
+
if verdict is None or verdict.get("status") != "admit":
|
|
16290
|
+
report(f"M2 the {source} source did not admit alone: {verdict}")
|
|
16291
|
+
return 1
|
|
16292
|
+
neither = triage(both, "--labels", "bug", "--issue", "7")
|
|
16293
|
+
if neither is None or neither.get("status") != "refuse":
|
|
16294
|
+
report(f"M2 an issue matching neither source was admitted: {neither}")
|
|
16295
|
+
return 1
|
|
16296
|
+
neither_reason = neither.get("reason") or ""
|
|
16297
|
+
for needle in ("bug", "7", "auto", "62"):
|
|
16298
|
+
if needle not in neither_reason:
|
|
16299
|
+
report(
|
|
16300
|
+
"M2 the refusal must name what the issue carried and both "
|
|
16301
|
+
f"halves of what is accepted; missing {needle}: {neither_reason}"
|
|
16302
|
+
)
|
|
16303
|
+
return 1
|
|
16304
|
+
|
|
16305
|
+
# M3 — a declaration Keel cannot fully read admits nothing, including
|
|
16306
|
+
# the half it did read, and names the part that failed.
|
|
16307
|
+
unreadable = {
|
|
16308
|
+
"hash": ("triage:\n issues:\n - '#62'\n labels:\n - auto\n", "#62"),
|
|
16309
|
+
"unknown-key": (
|
|
16310
|
+
"triage:\n authors:\n - someone\n labels:\n - auto\n",
|
|
16311
|
+
"authors",
|
|
16312
|
+
),
|
|
16313
|
+
"mixed": ("triage:\n - auto\n issues:\n - 62\n", "auto"),
|
|
16314
|
+
"inline": ("triage: { issues: [62] }\n", "issues"),
|
|
16315
|
+
}
|
|
16316
|
+
undeclared = make(root, "undeclared", "")
|
|
16317
|
+
undeclared_reason = (triage(undeclared, "--labels", "auto") or {}).get(
|
|
16318
|
+
"reason"
|
|
16319
|
+
) or ""
|
|
16320
|
+
for name, (body, needle) in unreadable.items():
|
|
16321
|
+
repo = make(root, name, body)
|
|
16322
|
+
# Every fixture above carries something Keel *did* read — the label
|
|
16323
|
+
# `auto`, or the number 62. A partial read would admit here.
|
|
16324
|
+
verdict = triage(repo, "--labels", "auto", "--issue", "62")
|
|
16325
|
+
if verdict is None or verdict.get("status") != "refuse":
|
|
16326
|
+
report(
|
|
16327
|
+
f"M3 the {name} declaration admitted on the half Keel could "
|
|
16328
|
+
f"read: {verdict}"
|
|
16329
|
+
)
|
|
16330
|
+
return 1
|
|
16331
|
+
reason = verdict.get("reason") or ""
|
|
16332
|
+
if needle not in reason:
|
|
16333
|
+
report(
|
|
16334
|
+
f"M3 the {name} refusal does not name the entry it could not "
|
|
16335
|
+
f"read ({needle}): {reason}"
|
|
16336
|
+
)
|
|
16337
|
+
return 1
|
|
16338
|
+
if reason == undeclared_reason:
|
|
16339
|
+
report(
|
|
16340
|
+
f"M3 the {name} refusal is word for word the undeclared-policy "
|
|
16341
|
+
"refusal, so a broken declaration reads as an absent one."
|
|
16342
|
+
)
|
|
16343
|
+
return 1
|
|
16344
|
+
if "could not" not in reason.lower():
|
|
16345
|
+
report(f"M3 the {name} refusal does not say it could not read: {reason}")
|
|
16346
|
+
return 1
|
|
16347
|
+
|
|
16348
|
+
# M3 — and the CLI refuses the same spelling it refuses in the file,
|
|
16349
|
+
# and refuses to answer with no issue attributes at all.
|
|
16350
|
+
hashed = run_keel(numbers, "triage", ".", "--issue", "#62")
|
|
16351
|
+
if hashed.returncode == 0:
|
|
16352
|
+
report("M3 --issue accepted a number Keel refuses in the declaration.")
|
|
16353
|
+
return 1
|
|
16354
|
+
empty = run_keel(numbers, "triage", ".")
|
|
16355
|
+
if empty.returncode == 0:
|
|
16356
|
+
report("M3 triage answered with neither labels nor an issue number.")
|
|
16357
|
+
return 1
|
|
16358
|
+
|
|
16359
|
+
# M4 — what may start work here is answerable from one command.
|
|
16360
|
+
doctor_both = run_keel(both, "--doctor").stdout
|
|
16361
|
+
if "Unattended triage:" not in doctor_both or "triage: ok" not in doctor_both:
|
|
16362
|
+
report(f"M4 doctor does not report a declared triage surface:\n{doctor_both}")
|
|
16363
|
+
return 1
|
|
16364
|
+
for needle in ("auto", "62"):
|
|
16365
|
+
if needle not in doctor_both:
|
|
16366
|
+
report(
|
|
16367
|
+
f"M4 doctor omits the declared {needle} source, so the surface "
|
|
16368
|
+
f"under-reports what may start work:\n{doctor_both}"
|
|
16369
|
+
)
|
|
16370
|
+
return 1
|
|
16371
|
+
doctor_numbers = run_keel(numbers, "--doctor").stdout
|
|
16372
|
+
if "triage: ok" not in doctor_numbers or "62" not in doctor_numbers:
|
|
16373
|
+
report(f"M4 doctor does not report an issues-only policy:\n{doctor_numbers}")
|
|
16374
|
+
return 1
|
|
16375
|
+
if "labelled" in doctor_numbers:
|
|
16376
|
+
report(
|
|
16377
|
+
"M4 doctor names labels for a repository that declared none:\n"
|
|
16378
|
+
f"{doctor_numbers}"
|
|
16379
|
+
)
|
|
16380
|
+
return 1
|
|
16381
|
+
doctor_broken = run_keel(root / "hash", "--doctor").stdout
|
|
16382
|
+
if "triage: ok" in doctor_broken:
|
|
16383
|
+
report(
|
|
16384
|
+
"M4 doctor reports an unreadable declaration as a working policy:\n"
|
|
16385
|
+
f"{doctor_broken}"
|
|
16386
|
+
)
|
|
16387
|
+
return 1
|
|
16388
|
+
if "#62" not in doctor_broken:
|
|
16389
|
+
report(
|
|
16390
|
+
"M4 doctor does not name what it could not read, so the owner "
|
|
16391
|
+
f"cannot tell a broken declaration from an absent one:\n{doctor_broken}"
|
|
16392
|
+
)
|
|
16393
|
+
return 1
|
|
16394
|
+
|
|
16395
|
+
# M5 — the surfaces an owner reads must state the second source. Phrases,
|
|
16396
|
+
# not keywords: "issue number" appearing anywhere would satisfy a keyword
|
|
16397
|
+
# check while the page still teaches that a label is the only unit. The
|
|
16398
|
+
# alignment skill points at the protocol's own statement instead of
|
|
16399
|
+
# repeating it (keel-unattended-triage's "A secondary surface points
|
|
16400
|
+
# instead of repeating" scenario), so only the three full surfaces below
|
|
16401
|
+
# are held to the phrase itself; the skill copies are held to the pointer.
|
|
16402
|
+
full_surfaces = {
|
|
16403
|
+
"config": ROOT / "keel/config.yaml",
|
|
16404
|
+
"protocol": ROOT / "AGENTS.md",
|
|
16405
|
+
"readme": ROOT / "README.md",
|
|
16406
|
+
}
|
|
16407
|
+
stale = "A label is the unit"
|
|
16408
|
+
for label, path in full_surfaces.items():
|
|
16409
|
+
if not path.is_file():
|
|
16410
|
+
report(f"M5 missing surface: {path}")
|
|
16411
|
+
return 1
|
|
16412
|
+
content = re.sub(r"\s+", " ", path.read_text(encoding="utf-8"))
|
|
16413
|
+
if "issue number" not in content:
|
|
16414
|
+
report(f"M5 {label} does not state the issue-number source: {path}")
|
|
16415
|
+
return 1
|
|
16416
|
+
if stale in content:
|
|
16417
|
+
report(
|
|
16418
|
+
f"M5 {label} still says {stale!r}, which stopped being true when "
|
|
16419
|
+
f"a second source was declared: {path}"
|
|
16420
|
+
)
|
|
16421
|
+
return 1
|
|
16422
|
+
|
|
16423
|
+
canonical = ROOT / "src/skills/keel-align-expectations/SKILL.md"
|
|
16424
|
+
distributed = ROOT / PLUGIN_ROOT / "skills/keel-align-expectations/SKILL.md"
|
|
16425
|
+
for label, path in (("canonical skill", canonical), ("distributed skill", distributed)):
|
|
16426
|
+
if not path.is_file():
|
|
16427
|
+
report(f"M5 missing surface: {path}")
|
|
16428
|
+
return 1
|
|
16429
|
+
content = re.sub(r"\s+", " ", path.read_text(encoding="utf-8"))
|
|
16430
|
+
if "AGENTS.md" not in content or "states no separate copy" not in content:
|
|
16431
|
+
report(
|
|
16432
|
+
f"M5 {label} does not point to the protocol's issue-number "
|
|
16433
|
+
f"statement: {path}"
|
|
16434
|
+
)
|
|
16435
|
+
return 1
|
|
16436
|
+
if canonical.read_bytes() != distributed.read_bytes():
|
|
16437
|
+
report("M5 the canonical and distributed skills diverged.")
|
|
16438
|
+
return 1
|
|
16439
|
+
|
|
16440
|
+
report("triage-admits-from-the-repository scenario passed.")
|
|
16441
|
+
return 0
|
|
16442
|
+
|
|
16443
|
+
|
|
14053
16444
|
def validate_delegation_declaration_scenario() -> int:
|
|
14054
16445
|
"""Who runs a task is a declaration, never an inference from its size.
|
|
14055
16446
|
|
|
@@ -15147,7 +17538,11 @@ def validate_unattended_boundary_scenario() -> int:
|
|
|
15147
17538
|
"""The boundary must be readable where an unattended run will read it.
|
|
15148
17539
|
|
|
15149
17540
|
Phrases, not keywords: "unattended" appearing somewhere would satisfy a
|
|
15150
|
-
keyword check while stating none of what a run may and may not do.
|
|
17541
|
+
keyword check while stating none of what a run may and may not do. The
|
|
17542
|
+
alignment skill points at the protocol's own statement instead of
|
|
17543
|
+
repeating it (keel-unattended-triage's "A secondary surface points
|
|
17544
|
+
instead of repeating" scenario), so only the protocol is held to the
|
|
17545
|
+
full phrase set; the skill copies are held to the pointer instead.
|
|
15151
17546
|
"""
|
|
15152
17547
|
|
|
15153
17548
|
required = [
|
|
@@ -15161,22 +17556,35 @@ def validate_unattended_boundary_scenario() -> int:
|
|
|
15161
17556
|
# Admission comes from a declaration, never from accumulated history.
|
|
15162
17557
|
"never from a precedent",
|
|
15163
17558
|
]
|
|
17559
|
+
pointer_required = [
|
|
17560
|
+
"AGENTS.md",
|
|
17561
|
+
"Unattended runs",
|
|
17562
|
+
"states no separate copy",
|
|
17563
|
+
]
|
|
15164
17564
|
canonical = ROOT / "src/skills/keel-align-expectations/SKILL.md"
|
|
15165
17565
|
distributed = ROOT / PLUGIN_ROOT / "skills/keel-align-expectations/SKILL.md"
|
|
15166
17566
|
protocol = ROOT / "AGENTS.md"
|
|
15167
17567
|
|
|
17568
|
+
if not protocol.is_file():
|
|
17569
|
+
report(f"unattended-boundary: missing protocol: {protocol}")
|
|
17570
|
+
return 1
|
|
17571
|
+
# Collapse whitespace: these are multi-word phrases in hard-wrapped
|
|
17572
|
+
# prose, so raw matching would assert the line layout, not the wording.
|
|
17573
|
+
protocol_content = re.sub(r"\s+", " ", protocol.read_text(encoding="utf-8"))
|
|
17574
|
+
for phrase in required:
|
|
17575
|
+
if phrase not in protocol_content:
|
|
17576
|
+
report(f"unattended-boundary: protocol omits: {phrase}")
|
|
17577
|
+
return 1
|
|
17578
|
+
|
|
15168
17579
|
for label, path in (
|
|
15169
|
-
("protocol", protocol),
|
|
15170
17580
|
("canonical skill", canonical),
|
|
15171
17581
|
("distributed skill", distributed),
|
|
15172
17582
|
):
|
|
15173
17583
|
if not path.is_file():
|
|
15174
17584
|
report(f"unattended-boundary: missing {label}: {path}")
|
|
15175
17585
|
return 1
|
|
15176
|
-
# Collapse whitespace: these are multi-word phrases in hard-wrapped
|
|
15177
|
-
# prose, so raw matching would assert the line layout, not the wording.
|
|
15178
17586
|
content = re.sub(r"\s+", " ", path.read_text(encoding="utf-8"))
|
|
15179
|
-
for phrase in
|
|
17587
|
+
for phrase in pointer_required:
|
|
15180
17588
|
if phrase not in content:
|
|
15181
17589
|
report(f"unattended-boundary: {label} omits: {phrase}")
|
|
15182
17590
|
return 1
|
|
@@ -19073,6 +21481,202 @@ def validate_guard_stale_manifest_scenario() -> int:
|
|
|
19073
21481
|
return 0
|
|
19074
21482
|
|
|
19075
21483
|
|
|
21484
|
+
# The message a reader is given when the state cannot be reauthorized.
|
|
21485
|
+
#
|
|
21486
|
+
# 5.26.0 recorded verbatim, so the parse-miss cell compares against the text as
|
|
21487
|
+
# it shipped rather than against whatever the code happens to produce.
|
|
21488
|
+
PARSE_MISS_PROBLEM = (
|
|
21489
|
+
"Guarded task demo#1.1 no longer resolves; reauthorize through "
|
|
21490
|
+
"`keel gate task-start` and `keel guard start`."
|
|
21491
|
+
)
|
|
21492
|
+
|
|
21493
|
+
|
|
21494
|
+
def guard_status_payload(repo: Path, cell: str) -> tuple[dict, str]:
|
|
21495
|
+
"""Return `guard status --json` as a payload plus its first problem message.
|
|
21496
|
+
|
|
21497
|
+
A run that produced nothing readable is reported here rather than handed
|
|
21498
|
+
back as an empty payload. Otherwise one condition downstream would guard two
|
|
21499
|
+
unrelated failures — the command did not run, and the status was wrong — and
|
|
21500
|
+
every caller would report the second, sending the reader to a state that was
|
|
21501
|
+
never observed. Returning `None` for the payload is what makes the two
|
|
21502
|
+
distinguishable at the call site instead of at the reader.
|
|
21503
|
+
"""
|
|
21504
|
+
result = run_keel(repo, "guard", "status", "--json")
|
|
21505
|
+
if not result.stdout.strip():
|
|
21506
|
+
report(
|
|
21507
|
+
f"guard-status-stale-manifest: {cell} `guard status --json` "
|
|
21508
|
+
f"produced no output (exit {result.returncode}). "
|
|
21509
|
+
f"{(result.stderr or '').strip()!r}"
|
|
21510
|
+
)
|
|
21511
|
+
return None, ""
|
|
21512
|
+
try:
|
|
21513
|
+
payload = json.loads(result.stdout)
|
|
21514
|
+
except json.JSONDecodeError as error:
|
|
21515
|
+
report(
|
|
21516
|
+
f"guard-status-stale-manifest: {cell} `guard status --json` "
|
|
21517
|
+
f"produced output that is not JSON ({error}): {result.stdout!r}"
|
|
21518
|
+
)
|
|
21519
|
+
return None, ""
|
|
21520
|
+
problems = payload.get("problems", [])
|
|
21521
|
+
return payload, (problems[0].get("message", "") if problems else "")
|
|
21522
|
+
|
|
21523
|
+
|
|
21524
|
+
def validate_guard_status_stale_manifest_scenario() -> int:
|
|
21525
|
+
"""`keel guard status` says which of two states the manifest is in.
|
|
21526
|
+
|
|
21527
|
+
`loadTaskContract` returns `null` when the tasks file is absent and when the
|
|
21528
|
+
task id is not in it. `guardStatus` had one branch for both, so a manifest
|
|
21529
|
+
whose change had been archived and a manifest whose task id was merely
|
|
21530
|
+
renumbered produced byte-identical output — and the shared message told the
|
|
21531
|
+
reader to reauthorize, which for the archived change is impossible:
|
|
21532
|
+
`keel gate task-start` reports a missing tasks file and `keel guard start`
|
|
21533
|
+
reports that the task does not exist. Neither names `keel guard clear`,
|
|
21534
|
+
which is the only action that resolves it.
|
|
21535
|
+
|
|
21536
|
+
The write guard hook drew this line in 5.12.0 by testing the change
|
|
21537
|
+
directory. This is the same test on the surface that describes the manifest
|
|
21538
|
+
rather than the one that refuses a write. Measured at the start of this
|
|
21539
|
+
change: both states returned `Guarded task demo#1.1 no longer resolves`.
|
|
21540
|
+
"""
|
|
21541
|
+
with tempfile.TemporaryDirectory(
|
|
21542
|
+
prefix="keel-guard-status-stale-", ignore_cleanup_errors=True
|
|
21543
|
+
) as raw_tmp:
|
|
21544
|
+
repo = Path(raw_tmp) / "repo"
|
|
21545
|
+
repo.mkdir()
|
|
21546
|
+
change_dir = repo / "openspec/changes/demo"
|
|
21547
|
+
write_text(change_dir / "tasks.md", guard_task_fixture())
|
|
21548
|
+
write_text(repo / "src/feature.js", "// fixture\n")
|
|
21549
|
+
|
|
21550
|
+
started = run_keel(
|
|
21551
|
+
repo, "guard", "start", "--change", "demo", "--task", "1.1", "--json"
|
|
21552
|
+
)
|
|
21553
|
+
if started.returncode != 0:
|
|
21554
|
+
report("guard-status-stale-manifest: guard start failed.")
|
|
21555
|
+
report((started.stderr or started.stdout).strip())
|
|
21556
|
+
return 1
|
|
21557
|
+
|
|
21558
|
+
# Relocated to where `openspec archive` moves it, for the reason the
|
|
21559
|
+
# guard-stale-manifest scenario records: what is under test is the
|
|
21560
|
+
# answer once the change is gone, not the archiver.
|
|
21561
|
+
archived_dir = repo / "openspec/changes/archive/2026-08-04-demo"
|
|
21562
|
+
archived_dir.parent.mkdir(parents=True, exist_ok=True)
|
|
21563
|
+
shutil.move(str(change_dir), str(archived_dir))
|
|
21564
|
+
|
|
21565
|
+
# M1 — the archived state names the directory that is gone and the
|
|
21566
|
+
# action that resolves it.
|
|
21567
|
+
stale, stale_message = guard_status_payload(repo, "M1")
|
|
21568
|
+
if stale is None:
|
|
21569
|
+
return 1
|
|
21570
|
+
if stale.get("status") != "drifted":
|
|
21571
|
+
report(
|
|
21572
|
+
"guard-status-stale-manifest: M1 expected the archived state to "
|
|
21573
|
+
f"stay classified as drifted, got {stale.get('status')!r}."
|
|
21574
|
+
)
|
|
21575
|
+
return 1
|
|
21576
|
+
stale_codes = [item.get("code") for item in stale.get("problems", [])]
|
|
21577
|
+
if "stale-manifest" not in stale_codes:
|
|
21578
|
+
report(
|
|
21579
|
+
"guard-status-stale-manifest: M1 an archived change was not "
|
|
21580
|
+
"reported under its own problem code, so a --json reader cannot "
|
|
21581
|
+
f"tell it from a parse miss. Codes: {stale_codes!r}"
|
|
21582
|
+
)
|
|
21583
|
+
return 1
|
|
21584
|
+
for needle in ("stale", "openspec/changes/demo", "keel guard clear"):
|
|
21585
|
+
if needle not in stale_message:
|
|
21586
|
+
report(
|
|
21587
|
+
"guard-status-stale-manifest: M1 the message must report the "
|
|
21588
|
+
f"manifest as stale and name what resolves it; {needle!r} is "
|
|
21589
|
+
f"missing from: {stale_message!r}"
|
|
21590
|
+
)
|
|
21591
|
+
return 1
|
|
21592
|
+
# The advice that could not be followed. `keel gate task-start` may
|
|
21593
|
+
# still be named for the task the reader is actually starting — what
|
|
21594
|
+
# must be gone is the instruction to reauthorize the vanished one.
|
|
21595
|
+
for forbidden in ("reauthorize through", "keel guard start"):
|
|
21596
|
+
if forbidden in stale_message:
|
|
21597
|
+
report(
|
|
21598
|
+
"guard-status-stale-manifest: M1 the message still sends the "
|
|
21599
|
+
f"reader to reauthorize a change that is not there; "
|
|
21600
|
+
f"{forbidden!r} is in: {stale_message!r}"
|
|
21601
|
+
)
|
|
21602
|
+
return 1
|
|
21603
|
+
|
|
21604
|
+
# M3 — the parse miss is untouched. A live change whose task id is
|
|
21605
|
+
# absent is not a fact about the repository, it is a read the guard
|
|
21606
|
+
# must not guess about, and its answer is the 5.26.0 answer verbatim.
|
|
21607
|
+
change_dir.mkdir(parents=True, exist_ok=True)
|
|
21608
|
+
write_text(
|
|
21609
|
+
change_dir / "tasks.md",
|
|
21610
|
+
guard_task_fixture().replace("1.1", "9.9"),
|
|
21611
|
+
)
|
|
21612
|
+
miss, miss_message = guard_status_payload(repo, "M3")
|
|
21613
|
+
if miss is None:
|
|
21614
|
+
return 1
|
|
21615
|
+
if miss.get("status") != "drifted":
|
|
21616
|
+
report(
|
|
21617
|
+
"guard-status-stale-manifest: M3 the parse miss changed status "
|
|
21618
|
+
f"from drifted to {miss.get('status')!r}."
|
|
21619
|
+
)
|
|
21620
|
+
return 1
|
|
21621
|
+
miss_codes = [item.get("code") for item in miss.get("problems", [])]
|
|
21622
|
+
if miss_codes != ["authority-drift"]:
|
|
21623
|
+
report(
|
|
21624
|
+
"guard-status-stale-manifest: M3 the parse miss changed problem "
|
|
21625
|
+
f"code. Expected ['authority-drift'], got {miss_codes!r}"
|
|
21626
|
+
)
|
|
21627
|
+
return 1
|
|
21628
|
+
if miss_message != PARSE_MISS_PROBLEM:
|
|
21629
|
+
report(
|
|
21630
|
+
"guard-status-stale-manifest: M3 the parse-miss message changed "
|
|
21631
|
+
f"from the 5.26.0 text.\nwas: {PARSE_MISS_PROBLEM!r}\nnow: "
|
|
21632
|
+
f"{miss_message!r}"
|
|
21633
|
+
)
|
|
21634
|
+
return 1
|
|
21635
|
+
|
|
21636
|
+
# M2 — the two states no longer answer the same. Asserted on both the
|
|
21637
|
+
# code and the message: a repair that reworded one branch while leaving
|
|
21638
|
+
# one code for two states would pass a message comparison alone.
|
|
21639
|
+
if stale_codes == miss_codes:
|
|
21640
|
+
report(
|
|
21641
|
+
"guard-status-stale-manifest: M2 an archived change and a task "
|
|
21642
|
+
"id absent from a live tasks.md still share a problem code "
|
|
21643
|
+
f"({stale_codes!r}), so they are separable only by prose."
|
|
21644
|
+
)
|
|
21645
|
+
return 1
|
|
21646
|
+
if stale_message == miss_message:
|
|
21647
|
+
report(
|
|
21648
|
+
"guard-status-stale-manifest: M2 the two states still produce "
|
|
21649
|
+
f"one message: {stale_message!r}"
|
|
21650
|
+
)
|
|
21651
|
+
return 1
|
|
21652
|
+
|
|
21653
|
+
# D4 — a third state: the change directory is there and its tasks file
|
|
21654
|
+
# is not. That is a live change mid-authoring, where reauthorizing after
|
|
21655
|
+
# writing the tasks file is genuinely the way out, so it keeps the
|
|
21656
|
+
# parse-miss message. This is what testing the directory rather than the
|
|
21657
|
+
# tasks file buys, and it is the cell that fails if the two are swapped.
|
|
21658
|
+
(change_dir / "tasks.md").unlink()
|
|
21659
|
+
authoring, authoring_message = guard_status_payload(repo, "D4")
|
|
21660
|
+
if authoring is None:
|
|
21661
|
+
return 1
|
|
21662
|
+
if authoring_message != PARSE_MISS_PROBLEM:
|
|
21663
|
+
report(
|
|
21664
|
+
"guard-status-stale-manifest: D4 a live change whose tasks.md "
|
|
21665
|
+
"is absent was reported as a stale manifest; the change is "
|
|
21666
|
+
f"still there and can be reauthorized. Got: {authoring_message!r}"
|
|
21667
|
+
)
|
|
21668
|
+
return 1
|
|
21669
|
+
|
|
21670
|
+
if "guard-status-stale-manifest" not in {name for name, _ in SCENARIOS}:
|
|
21671
|
+
report(
|
|
21672
|
+
"guard-status-stale-manifest: the scenario registry does not "
|
|
21673
|
+
"include it."
|
|
21674
|
+
)
|
|
21675
|
+
return 1
|
|
21676
|
+
report("guard-status-stale-manifest scenario passed.")
|
|
21677
|
+
return 0
|
|
21678
|
+
|
|
21679
|
+
|
|
19076
21680
|
# The one-message-many-failures shape, counted rather than forbidden.
|
|
19077
21681
|
#
|
|
19078
21682
|
# Issue #43 records it caught by `keel-review-checklist` in three consecutive
|
|
@@ -19096,7 +21700,15 @@ def validate_guard_stale_manifest_scenario() -> int:
|
|
|
19096
21700
|
# lowered, because a number that only checks for rises becomes false the first
|
|
19097
21701
|
# time someone fixes a site — and a false number is what this whole change is
|
|
19098
21702
|
# about. Fixing sites is expected. Lowering this constant is one line.
|
|
19099
|
-
|
|
21703
|
+
#
|
|
21704
|
+
# Raised 75 -> 80 by `change-verify-deferred-evidence` (issue #95): five sites,
|
|
21705
|
+
# each `payload.get("status") != "fail" or "<code>" not in codes(payload)`
|
|
21706
|
+
# guarding one `report(...)`, the same already-accepted gate-result-assertion
|
|
21707
|
+
# shape as `regression-check-tag`'s own site above. The message covers both
|
|
21708
|
+
# operands (the gate did not fail the way the fixture requires, whichever half
|
|
21709
|
+
# was wrong), and every site prints the full problem payload alongside it, so
|
|
21710
|
+
# a reader is never left with only the sentence.
|
|
21711
|
+
OR_GUARDED_ASSERTION_SITES = 80
|
|
19100
21712
|
|
|
19101
21713
|
|
|
19102
21714
|
def or_guarded_assertion_sites(source: str) -> list[int]:
|
|
@@ -19344,10 +21956,698 @@ def validate_assertion_shape_count_scenario() -> int:
|
|
|
19344
21956
|
return 0
|
|
19345
21957
|
|
|
19346
21958
|
|
|
19347
|
-
def
|
|
19348
|
-
|
|
19349
|
-
|
|
19350
|
-
|
|
21959
|
+
def validate_declared_paths_are_read_whole_scenario() -> int:
|
|
21960
|
+
"""Issue #60: a durable owner under a Chinese directory was refused.
|
|
21961
|
+
|
|
21962
|
+
`notes/note-006-转岗最难的不是流程/note.md` exists and `git ls-files` finds
|
|
21963
|
+
it, and `change-close` reported that `notes/note-006-` does not exist — a
|
|
21964
|
+
path nobody wrote. The extractor was `[A-Za-z0-9._-]+(?:/[A-Za-z0-9._-]+)+`,
|
|
21965
|
+
which answers "where does the path end" by assuming what a path is made of.
|
|
21966
|
+
|
|
21967
|
+
Two more shapes fell out while reproducing it: a path whose *first* segment
|
|
21968
|
+
is not ASCII does not match at all, and a path containing a space truncates
|
|
21969
|
+
at the space. The last is why the backtick form matters — `Touch` already
|
|
21970
|
+
accepts it, so one authorship was spelled two ways depending on which
|
|
21971
|
+
reader would read it.
|
|
21972
|
+
|
|
21973
|
+
Same class as #40, which fixed it in `gitPaths` for the worktree side. That
|
|
21974
|
+
fix did not generalize because it repaired one reader rather than how the
|
|
21975
|
+
repository extracts paths.
|
|
21976
|
+
"""
|
|
21977
|
+
label = "declared-paths-are-read-whole"
|
|
21978
|
+
cjk_owner = "notes/note-006-转岗最难的不是流程/note.md"
|
|
21979
|
+
cjk_first = "文档/风格.md"
|
|
21980
|
+
spaced = "docs/has space.md"
|
|
21981
|
+
|
|
21982
|
+
def git(repo, *args):
|
|
21983
|
+
return subprocess.run(
|
|
21984
|
+
["git", "-C", str(repo), *args], capture_output=True, text=True
|
|
21985
|
+
)
|
|
21986
|
+
|
|
21987
|
+
def tasks_doc(findings: str, coverage: str) -> str:
|
|
21988
|
+
return (
|
|
21989
|
+
"# Tasks\n\n"
|
|
21990
|
+
"- [x] 1.1 Exercise the declared-path readers\n"
|
|
21991
|
+
" - Covers:\n - E1: Public behavior passes.\n"
|
|
21992
|
+
" - Touch:\n - src/declared.js\n"
|
|
21993
|
+
" - Verify:\n - Strategy: evidence-first\n"
|
|
21994
|
+
" - M1: node test.js asserts the recorded feed status\n"
|
|
21995
|
+
" - Evidence:\n"
|
|
21996
|
+
" - Contract: keel-task-capsule/v1 sha256:"
|
|
21997
|
+
+ ("0" * 64) + "\n"
|
|
21998
|
+
" - M1: the suite passed\n"
|
|
21999
|
+
" - Review:\n - Status: pass\n"
|
|
22000
|
+
" - Acceptance check: behavior asserted at the interface\n"
|
|
22001
|
+
" - Scope check: only Touch files changed\n"
|
|
22002
|
+
f" - Findings: {findings}\n"
|
|
22003
|
+
" - Blocker: none\n\n"
|
|
22004
|
+
"## Invalidates\n\n- None.\n\n"
|
|
22005
|
+
"## Expectation Coverage\n\n"
|
|
22006
|
+
f"- E1: the behavior {coverage}\n"
|
|
22007
|
+
)
|
|
22008
|
+
|
|
22009
|
+
with tempfile.TemporaryDirectory(prefix="keel-declared-paths-") as raw:
|
|
22010
|
+
repo = (Path(raw) / "repo").resolve()
|
|
22011
|
+
repo.mkdir()
|
|
22012
|
+
git(repo, "init", "-q")
|
|
22013
|
+
git(repo, "config", "user.email", "t@example.com")
|
|
22014
|
+
git(repo, "config", "user.name", "keel-test")
|
|
22015
|
+
for item in (cjk_owner, cjk_first, spaced):
|
|
22016
|
+
write_text(repo / item, "# owner\n")
|
|
22017
|
+
write_text(repo / "src/declared.js", "// product\n")
|
|
22018
|
+
tasks_path = repo / "openspec/changes/demo/tasks.md"
|
|
22019
|
+
|
|
22020
|
+
def close(findings: str, coverage: str) -> dict:
|
|
22021
|
+
write_text(tasks_path, tasks_doc(findings, coverage))
|
|
22022
|
+
return json.loads(
|
|
22023
|
+
run_keel(
|
|
22024
|
+
repo, "gate", "change-close", "--change", "demo",
|
|
22025
|
+
"--action", "sync", "--json",
|
|
22026
|
+
).stdout
|
|
22027
|
+
)
|
|
22028
|
+
|
|
22029
|
+
def problems(payload: dict) -> list[str]:
|
|
22030
|
+
return [p.get("message", "") for p in payload.get("problems", [])]
|
|
22031
|
+
|
|
22032
|
+
# M1 — the reported shape, on both readers that take a declared path.
|
|
22033
|
+
for where, findings, coverage in (
|
|
22034
|
+
("Findings", f"one open. Durable owner: {cjk_owner}", "Covered by: 1.1"),
|
|
22035
|
+
("Expectation Coverage", "none", f"Durable owner: {cjk_owner}"),
|
|
22036
|
+
):
|
|
22037
|
+
payload = close(findings, coverage)
|
|
22038
|
+
truncated = [m for m in problems(payload) if "note-006-" in m]
|
|
22039
|
+
if truncated:
|
|
22040
|
+
report(
|
|
22041
|
+
f"{label} M1 a {where} durable owner naming an existing "
|
|
22042
|
+
"file under a non-ASCII directory was refused, and the "
|
|
22043
|
+
"refusal names a path nobody wrote."
|
|
22044
|
+
)
|
|
22045
|
+
for message in truncated:
|
|
22046
|
+
report(f" {message}")
|
|
22047
|
+
return 1
|
|
22048
|
+
|
|
22049
|
+
# M1 — a path whose first segment is not ASCII matched nothing at all,
|
|
22050
|
+
# which is a different failure from truncation and needs its own case.
|
|
22051
|
+
payload = close("none", f"Durable owner: {cjk_first}")
|
|
22052
|
+
rejected = [m for m in problems(payload) if "E1" in m and "owner" in m.lower()]
|
|
22053
|
+
if rejected:
|
|
22054
|
+
report(
|
|
22055
|
+
f"{label} M1 a durable owner whose first segment is not ASCII "
|
|
22056
|
+
"was not recognized as a path at all."
|
|
22057
|
+
)
|
|
22058
|
+
for message in rejected:
|
|
22059
|
+
report(f" {message}")
|
|
22060
|
+
return 1
|
|
22061
|
+
|
|
22062
|
+
# M1 — whitespace arrives in backticks, the form Touch already accepts.
|
|
22063
|
+
payload = close("none", f"Durable owner: `{spaced}`")
|
|
22064
|
+
rejected = [m for m in problems(payload) if "E1" in m and "owner" in m.lower()]
|
|
22065
|
+
if rejected:
|
|
22066
|
+
report(
|
|
22067
|
+
f"{label} M1 a backtick-wrapped path containing a space was "
|
|
22068
|
+
"not read whole, so Touch and Durable owner still disagree "
|
|
22069
|
+
"about how one path is written."
|
|
22070
|
+
)
|
|
22071
|
+
for message in rejected:
|
|
22072
|
+
report(f" {message}")
|
|
22073
|
+
return 1
|
|
22074
|
+
|
|
22075
|
+
# M1 — a path ending a sentence, in both punctuation families.
|
|
22076
|
+
for mark in ("。", ","):
|
|
22077
|
+
payload = close("none", f"Durable owner: {cjk_owner}{mark} 说明文字")
|
|
22078
|
+
rejected = [
|
|
22079
|
+
m for m in problems(payload) if "E1" in m and "owner" in m.lower()
|
|
22080
|
+
]
|
|
22081
|
+
if rejected:
|
|
22082
|
+
report(
|
|
22083
|
+
f"{label} M1 a path followed by {mark!r} was extended by "
|
|
22084
|
+
"its punctuation, so the gate looked for a file that "
|
|
22085
|
+
"cannot exist."
|
|
22086
|
+
)
|
|
22087
|
+
for message in rejected:
|
|
22088
|
+
report(f" {message}")
|
|
22089
|
+
return 1
|
|
22090
|
+
|
|
22091
|
+
# M1 — the check itself must survive the widening. A path that does not
|
|
22092
|
+
# exist is still refused, and the refusal names the whole path.
|
|
22093
|
+
missing = "notes/不存在的目录/note.md"
|
|
22094
|
+
payload = close("none", f"Durable owner: {missing}")
|
|
22095
|
+
named = [m for m in problems(payload) if missing in m]
|
|
22096
|
+
if not named:
|
|
22097
|
+
report(
|
|
22098
|
+
f"{label} M1 a durable owner naming a file that does not exist "
|
|
22099
|
+
"was accepted, or was refused without naming the whole path. "
|
|
22100
|
+
"Widening the extractor must not weaken the existence check."
|
|
22101
|
+
)
|
|
22102
|
+
for message in problems(payload):
|
|
22103
|
+
report(f" {message}")
|
|
22104
|
+
return 1
|
|
22105
|
+
|
|
22106
|
+
if label not in {name for name, _ in SCENARIOS}:
|
|
22107
|
+
report(f"{label}: the scenario registry does not include it.")
|
|
22108
|
+
return 1
|
|
22109
|
+
report(f"{label} scenario passed.")
|
|
22110
|
+
return 0
|
|
22111
|
+
|
|
22112
|
+
|
|
22113
|
+
def validate_published_specs_validate_strictly_scenario() -> int:
|
|
22114
|
+
"""Issue #46: the store Keel publishes was validated by nothing.
|
|
22115
|
+
|
|
22116
|
+
Every change's closing task runs `openspec validate <change> --strict`,
|
|
22117
|
+
which reads the change directory and stays green. The published store under
|
|
22118
|
+
`openspec/specs/` was read by no check at all — measured at 5.16.0,
|
|
22119
|
+
`--specs` appeared zero times in this file. Issue #46 found 8 of 21 specs
|
|
22120
|
+
failing strict validation, every one on `Requirement must contain SHALL or
|
|
22121
|
+
MUST keyword`, because the requirement opened with a context paragraph and
|
|
22122
|
+
the strict validator reads only the block under the heading. Those failures
|
|
22123
|
+
had never appeared in front of anyone.
|
|
22124
|
+
|
|
22125
|
+
They are gone now, removed by spec rewrites made for other reasons. That is
|
|
22126
|
+
what makes this the moment to assert it absolutely rather than as the
|
|
22127
|
+
ratchet #46 proposed: at a count of 8 a ratchet was the honest shape, and at
|
|
22128
|
+
0 the same mechanism becomes a budget for failures to hide in.
|
|
22129
|
+
"""
|
|
22130
|
+
label = "published-specs-validate-strictly"
|
|
22131
|
+
|
|
22132
|
+
# The pinned binary, not whatever `openspec` PATH happens to offer. This is
|
|
22133
|
+
# not fussiness: measured here, PATH answers 1.4.1 and reports 8 failures
|
|
22134
|
+
# while the version this repository resolves answers 1.6.0 and reports
|
|
22135
|
+
# none. Issue #46 recorded those 8 failures as an openspec 1.6.0 result;
|
|
22136
|
+
# they are 1.4.1's, and the store has always passed under the version the
|
|
22137
|
+
# repository pins. A check that reads PATH would re-record the same
|
|
22138
|
+
# mistake, which is exactly what `keel-target-surface-diagnostics` means by
|
|
22139
|
+
# a suite that silently changes which program it runs reporting facts about
|
|
22140
|
+
# a different program.
|
|
22141
|
+
pinned = ROOT / "node_modules" / ".bin" / (
|
|
22142
|
+
"openspec.cmd" if os.name == "nt" else "openspec"
|
|
22143
|
+
)
|
|
22144
|
+
if not pinned.exists():
|
|
22145
|
+
report(
|
|
22146
|
+
f"{label}: the pinned openspec is not installed "
|
|
22147
|
+
"(node_modules/.bin/openspec); reporting the skip rather than "
|
|
22148
|
+
"falling back to PATH, which would answer for a different program."
|
|
22149
|
+
)
|
|
22150
|
+
return 0
|
|
22151
|
+
|
|
22152
|
+
def pinned_openspec(*args: str) -> subprocess.CompletedProcess[str]:
|
|
22153
|
+
return subprocess.run(
|
|
22154
|
+
[str(pinned), *args],
|
|
22155
|
+
cwd=ROOT,
|
|
22156
|
+
text=True,
|
|
22157
|
+
encoding="utf-8",
|
|
22158
|
+
errors="replace",
|
|
22159
|
+
capture_output=True,
|
|
22160
|
+
check=False,
|
|
22161
|
+
)
|
|
22162
|
+
|
|
22163
|
+
version = pinned_openspec("--version")
|
|
22164
|
+
exercised = re.search(r"\d+\.\d+\.\d+", version.stdout or "")
|
|
22165
|
+
result = pinned_openspec("validate", "--specs", "--strict")
|
|
22166
|
+
|
|
22167
|
+
output = f"{result.stdout or ''}{result.stderr or ''}"
|
|
22168
|
+
# Two independent readings. The exit status alone would pass if the command
|
|
22169
|
+
# stopped validating; the per-spec lines alone would pass if it started
|
|
22170
|
+
# exiting non-zero for an unrelated reason. Neither is trusted on its own.
|
|
22171
|
+
failed = [
|
|
22172
|
+
line.strip()
|
|
22173
|
+
for line in output.splitlines()
|
|
22174
|
+
if line.strip().startswith("✗")
|
|
22175
|
+
]
|
|
22176
|
+
totals = re.search(r"Totals:\s*(\d+) passed,\s*(\d+) failed", output)
|
|
22177
|
+
if not totals:
|
|
22178
|
+
report(
|
|
22179
|
+
f"{label} the validator produced no totals line, so the result "
|
|
22180
|
+
"cannot be read. The output shape it reports may have moved."
|
|
22181
|
+
)
|
|
22182
|
+
report(output.strip()[:600])
|
|
22183
|
+
return 1
|
|
22184
|
+
passed_count, failed_count = int(totals.group(1)), int(totals.group(2))
|
|
22185
|
+
|
|
22186
|
+
if failed:
|
|
22187
|
+
report(
|
|
22188
|
+
f"{label} {len(failed)} published spec(s) fail strict validation "
|
|
22189
|
+
f"against openspec {exercised.group(0) if exercised else 'unknown'}. "
|
|
22190
|
+
"A published spec that the validator Keel ships refuses is one "
|
|
22191
|
+
"every consumer sees refused. The usual cause is a requirement "
|
|
22192
|
+
"whose modal verb sits below its first paragraph — the strict "
|
|
22193
|
+
"validator reads only the block directly under the heading."
|
|
22194
|
+
)
|
|
22195
|
+
for line in failed:
|
|
22196
|
+
report(f" {line}")
|
|
22197
|
+
return 1
|
|
22198
|
+
if failed_count != 0:
|
|
22199
|
+
report(
|
|
22200
|
+
f"{label} the totals line reports {failed_count} failing spec(s) "
|
|
22201
|
+
"while no per-spec failure line was emitted. The two readings "
|
|
22202
|
+
"disagree, so the result is not trustworthy either way."
|
|
22203
|
+
)
|
|
22204
|
+
report(output.strip()[:600])
|
|
22205
|
+
return 1
|
|
22206
|
+
if result.returncode != 0:
|
|
22207
|
+
report(
|
|
22208
|
+
f"{label} the validator reported {passed_count} passed and 0 "
|
|
22209
|
+
f"failed but exited {result.returncode}. The verdict and the exit "
|
|
22210
|
+
"status disagree."
|
|
22211
|
+
)
|
|
22212
|
+
report(output.strip()[:600])
|
|
22213
|
+
return 1
|
|
22214
|
+
if passed_count == 0:
|
|
22215
|
+
report(
|
|
22216
|
+
f"{label} the validator reports zero specs. An empty store passes "
|
|
22217
|
+
"every assertion about it, which is the shape "
|
|
22218
|
+
"`A derived assertion set that collapses to empty fails instead of "
|
|
22219
|
+
"passing` exists to refuse."
|
|
22220
|
+
)
|
|
22221
|
+
return 1
|
|
22222
|
+
|
|
22223
|
+
if label not in {name for name, _ in SCENARIOS}:
|
|
22224
|
+
report(f"{label}: the scenario registry does not include it.")
|
|
22225
|
+
return 1
|
|
22226
|
+
report(
|
|
22227
|
+
f"{label} scenario passed: {passed_count} published specs validate "
|
|
22228
|
+
f"strictly against openspec "
|
|
22229
|
+
f"{exercised.group(0) if exercised else 'unknown'}."
|
|
22230
|
+
)
|
|
22231
|
+
return 0
|
|
22232
|
+
|
|
22233
|
+
|
|
22234
|
+
def validate_decimal_runs_are_not_hash_shaped_scenario() -> int:
|
|
22235
|
+
"""Issue #58: an eleven-digit fake phone number failed `keel state`.
|
|
22236
|
+
|
|
22237
|
+
The criterion was a context word beside `[0-9a-f]{7,40}`, and eleven
|
|
22238
|
+
decimal digits are eleven characters of that class. So was a timestamp,
|
|
22239
|
+
an order number, a port, and any numeric fixture that happened to sit on a
|
|
22240
|
+
line with the word `commit` or `提交` — which in evidence prose is most of
|
|
22241
|
+
them.
|
|
22242
|
+
|
|
22243
|
+
The cost is not the refusal, it is what the refusal asks for. Nothing is
|
|
22244
|
+
wrong with the line, so the only way past it is to write the evidence
|
|
22245
|
+
differently; the reporter changed the number to `138****0000`. Evidence
|
|
22246
|
+
reworded to satisfy a pattern is weaker than the evidence that was true,
|
|
22247
|
+
and a check that fails on correct work is one people learn to route
|
|
22248
|
+
around.
|
|
22249
|
+
|
|
22250
|
+
Same class as #60, repaired one release earlier in the other direction:
|
|
22251
|
+
there the character class was too narrow. The rule was right both times.
|
|
22252
|
+
"""
|
|
22253
|
+
label = "decimal-runs-are-not-hash-shaped"
|
|
22254
|
+
|
|
22255
|
+
def tasks_doc(body: str) -> str:
|
|
22256
|
+
return (
|
|
22257
|
+
"# Tasks\n\n## Invalidates\n\n- None.\n\n"
|
|
22258
|
+
"## Tasks\n\n"
|
|
22259
|
+
"- [x] A1 implementation\n"
|
|
22260
|
+
f"{body}\n"
|
|
22261
|
+
"## Workflow Notes\n\n- None.\n"
|
|
22262
|
+
)
|
|
22263
|
+
|
|
22264
|
+
# F2, measured at 5.18.0: each of these is refused by the pre-change
|
|
22265
|
+
# criterion, and none of them records anything git owns.
|
|
22266
|
+
accepted = (
|
|
22267
|
+
"- M1: pass —— 提交表单时手机号 13800138000 通过校验。\n",
|
|
22268
|
+
"- M2: pass —— 时间戳 1700000000 与 commit 记录对齐。\n",
|
|
22269
|
+
"- M3: pass —— 提交订单号 20260802123 落库。\n",
|
|
22270
|
+
)
|
|
22271
|
+
# The other half of the requirement: what the rule exists to refuse.
|
|
22272
|
+
refused_token = "- M4: pass —— 合入前的 commit a1b2c3d4e5f6 已验证。\n"
|
|
22273
|
+
# Wording alone, carrying no hash-shaped token at all.
|
|
22274
|
+
refused_wording = "- M5: 该任务**未提交**,等待评审。\n"
|
|
22275
|
+
|
|
22276
|
+
def state_of(check) -> str:
|
|
22277
|
+
if "keel state: ok" in check.stdout:
|
|
22278
|
+
return "ok"
|
|
22279
|
+
if "keel state: failed" in check.stdout:
|
|
22280
|
+
return "failed"
|
|
22281
|
+
return "unreported"
|
|
22282
|
+
|
|
22283
|
+
with tempfile.TemporaryDirectory(prefix="keel-decimal-runs-") as raw:
|
|
22284
|
+
repo = (Path(raw) / "repo").resolve()
|
|
22285
|
+
repo.mkdir()
|
|
22286
|
+
install = run_keel(repo, "--install")
|
|
22287
|
+
if install.returncode != 0:
|
|
22288
|
+
report(f"{label}: keel --install failed while building the fixture.")
|
|
22289
|
+
report((install.stderr or install.stdout).strip())
|
|
22290
|
+
return 1
|
|
22291
|
+
|
|
22292
|
+
tasks_path = repo / "openspec/changes/numbers-in-evidence/tasks.md"
|
|
22293
|
+
|
|
22294
|
+
def check(body: str):
|
|
22295
|
+
write_text(tasks_path, tasks_doc(body))
|
|
22296
|
+
return run_keel(repo, "--check")
|
|
22297
|
+
|
|
22298
|
+
# `keel state` reporting nothing at all is a different failure from it
|
|
22299
|
+
# reporting a refusal, and one condition covering both would send the
|
|
22300
|
+
# reader to a line that has no problem in it.
|
|
22301
|
+
def unreported(result, where: str) -> bool:
|
|
22302
|
+
if state_of(result) != "unreported":
|
|
22303
|
+
return False
|
|
22304
|
+
report(
|
|
22305
|
+
f"{label}: keel --check reported no state at all while {where}. "
|
|
22306
|
+
"This is not a verdict about the fixture — the check did not "
|
|
22307
|
+
"reach the point of having one."
|
|
22308
|
+
)
|
|
22309
|
+
report((result.stderr or result.stdout).strip()[:600])
|
|
22310
|
+
return True
|
|
22311
|
+
|
|
22312
|
+
# M1 — the reported shape and its two siblings, each on its own line so
|
|
22313
|
+
# a failure names which one.
|
|
22314
|
+
for line in accepted:
|
|
22315
|
+
result = check(line)
|
|
22316
|
+
if unreported(result, "reading one ordinary number"):
|
|
22317
|
+
return 1
|
|
22318
|
+
if state_of(result) != "ok":
|
|
22319
|
+
report(
|
|
22320
|
+
f"{label}: an ordinary number in evidence prose was refused "
|
|
22321
|
+
"as a recorded identifier. Nothing on this line records "
|
|
22322
|
+
"anything git owns, so the only way past the refusal is to "
|
|
22323
|
+
"reword evidence that was true."
|
|
22324
|
+
)
|
|
22325
|
+
report(f" line: {line.strip()}")
|
|
22326
|
+
for state_error in [
|
|
22327
|
+
out for out in result.stdout.splitlines()
|
|
22328
|
+
if out.startswith("state-error")
|
|
22329
|
+
]:
|
|
22330
|
+
report(f" {state_error}")
|
|
22331
|
+
return 1
|
|
22332
|
+
|
|
22333
|
+
# All three together, because the check reports per line and a
|
|
22334
|
+
# per-line pass says nothing about a file holding several.
|
|
22335
|
+
together = check("".join(accepted))
|
|
22336
|
+
if unreported(together, "reading three ordinary numbers in one file"):
|
|
22337
|
+
return 1
|
|
22338
|
+
if state_of(together) != "ok":
|
|
22339
|
+
report(f"{label}: three ordinary numbers in one file were refused.")
|
|
22340
|
+
report(together.stdout.strip()[:600])
|
|
22341
|
+
return 1
|
|
22342
|
+
|
|
22343
|
+
# M1 negative — the narrowing must not have narrowed the rule.
|
|
22344
|
+
with_token = check("".join(accepted) + refused_token)
|
|
22345
|
+
if unreported(with_token, "reading a hexadecimal identifier"):
|
|
22346
|
+
return 1
|
|
22347
|
+
if state_of(with_token) != "failed":
|
|
22348
|
+
report(
|
|
22349
|
+
f"{label}: a hexadecimal identifier of that length beside a "
|
|
22350
|
+
"context word was accepted. Narrowing what counts as an "
|
|
22351
|
+
"identifier must not stop the check refusing one."
|
|
22352
|
+
)
|
|
22353
|
+
report(with_token.stdout.strip()[:600])
|
|
22354
|
+
return 1
|
|
22355
|
+
if with_token.returncode == 0:
|
|
22356
|
+
report(f"{label}: the refusal did not fail the check's exit status.")
|
|
22357
|
+
return 1
|
|
22358
|
+
named = [
|
|
22359
|
+
out for out in with_token.stdout.splitlines()
|
|
22360
|
+
if out.startswith("state-error") and "tasks.md:" in out
|
|
22361
|
+
]
|
|
22362
|
+
if not named:
|
|
22363
|
+
report(
|
|
22364
|
+
f"{label}: the refusal named no line. An author cannot act on "
|
|
22365
|
+
"a refusal that does not say where."
|
|
22366
|
+
)
|
|
22367
|
+
report(with_token.stdout.strip()[:600])
|
|
22368
|
+
return 1
|
|
22369
|
+
|
|
22370
|
+
# And the wording patterns, which never depended on a digit run.
|
|
22371
|
+
wording = check(refused_wording)
|
|
22372
|
+
if unreported(wording, "reading recorded work state written in words"):
|
|
22373
|
+
return 1
|
|
22374
|
+
if state_of(wording) != "failed":
|
|
22375
|
+
report(
|
|
22376
|
+
f"{label}: recorded work state written in words was accepted. "
|
|
22377
|
+
"That rule reads the wording and is untouched by any change to "
|
|
22378
|
+
"what counts as an identifier."
|
|
22379
|
+
)
|
|
22380
|
+
report(wording.stdout.strip()[:600])
|
|
22381
|
+
return 1
|
|
22382
|
+
|
|
22383
|
+
if label not in {name for name, _ in SCENARIOS}:
|
|
22384
|
+
report(f"{label}: the scenario registry does not include it.")
|
|
22385
|
+
return 1
|
|
22386
|
+
report(f"{label} scenario passed.")
|
|
22387
|
+
return 0
|
|
22388
|
+
|
|
22389
|
+
|
|
22390
|
+
def _tasks_semantics_probe(raw: str):
|
|
22391
|
+
"""Install into `raw` and return `check(body) -> (state, errors)`.
|
|
22392
|
+
|
|
22393
|
+
Both scenarios below ask the same question of `keel --check` — which lines
|
|
22394
|
+
of one active `tasks.md` it refuses — so they share the fixture rather than
|
|
22395
|
+
each growing their own copy of it. The caller owns the temporary directory,
|
|
22396
|
+
so a failed install is reported by the caller and cleaned up by its `with`.
|
|
22397
|
+
"""
|
|
22398
|
+
repo = (Path(raw) / "repo").resolve()
|
|
22399
|
+
repo.mkdir()
|
|
22400
|
+
install = run_keel(repo, "--install")
|
|
22401
|
+
if install.returncode != 0:
|
|
22402
|
+
return None, (install.stderr or install.stdout).strip()
|
|
22403
|
+
|
|
22404
|
+
tasks_path = repo / "openspec/changes/reading-lines/tasks.md"
|
|
22405
|
+
|
|
22406
|
+
def check(body: str):
|
|
22407
|
+
write_text(
|
|
22408
|
+
tasks_path,
|
|
22409
|
+
"# Tasks\n\n## Invalidates\n\n- None.\n\n"
|
|
22410
|
+
"## Tasks\n\n"
|
|
22411
|
+
f"{body}\n"
|
|
22412
|
+
"## Workflow Notes\n\n- None.\n",
|
|
22413
|
+
)
|
|
22414
|
+
result = run_keel(repo, "--check")
|
|
22415
|
+
if "keel state: ok" in result.stdout:
|
|
22416
|
+
state = "ok"
|
|
22417
|
+
elif "keel state: failed" in result.stdout:
|
|
22418
|
+
state = "failed"
|
|
22419
|
+
else:
|
|
22420
|
+
state = "unreported"
|
|
22421
|
+
errors = [
|
|
22422
|
+
out for out in result.stdout.splitlines() if out.startswith("state-error")
|
|
22423
|
+
]
|
|
22424
|
+
return state, errors
|
|
22425
|
+
|
|
22426
|
+
return check, None
|
|
22427
|
+
|
|
22428
|
+
|
|
22429
|
+
def validate_a_context_word_is_a_word_scenario() -> int:
|
|
22430
|
+
"""Issue #65 §1: `remaining` supplied the word `main`.
|
|
22431
|
+
|
|
22432
|
+
The context words that turn a hash-shaped token into a recorded commit
|
|
22433
|
+
identifier were matched anywhere on the line, so `remaining`, `heading`,
|
|
22434
|
+
`domain`, and `maintains` each supplied one. Four lines in this
|
|
22435
|
+
repository's own history are refused this way, and every hex token on
|
|
22436
|
+
them is a `sha256:` contract anchor rather than a commit identifier.
|
|
22437
|
+
|
|
22438
|
+
The boundary form is the owner's decision on #65 and keeps the inflections
|
|
22439
|
+
an author actually writes. Dropping `committed` and `hashes` would have
|
|
22440
|
+
been the quieter change and would have cost four real refusals.
|
|
22441
|
+
"""
|
|
22442
|
+
label = "a-context-word-is-a-word"
|
|
22443
|
+
with tempfile.TemporaryDirectory(prefix="keel-context-word-") as raw:
|
|
22444
|
+
check, failure = _tasks_semantics_probe(raw)
|
|
22445
|
+
if failure is not None:
|
|
22446
|
+
report(f"{label}: keel --install failed while building the fixture.")
|
|
22447
|
+
report(failure)
|
|
22448
|
+
return 1
|
|
22449
|
+
|
|
22450
|
+
# A word is not the words inside it. Each carries a hash-shaped token so
|
|
22451
|
+
# that the host word is the only thing under test.
|
|
22452
|
+
for host in ("remaining", "heading", "domain", "maintains"):
|
|
22453
|
+
state, errors = check(
|
|
22454
|
+
"- [x] A1 implementation\n"
|
|
22455
|
+
f" - Evidence:\n"
|
|
22456
|
+
f" - M1: pass — 0 anchors {host}, recorded at sha256:01b9e740ab.\n"
|
|
22457
|
+
)
|
|
22458
|
+
if state == "unreported":
|
|
22459
|
+
report(f"{label}: keel --check reported no state at all on `{host}`.")
|
|
22460
|
+
return 1
|
|
22461
|
+
if state != "ok":
|
|
22462
|
+
report(
|
|
22463
|
+
f"{label}: `{host}` was read as a context word. It contains "
|
|
22464
|
+
"one, which is not the same thing, and the token beside it "
|
|
22465
|
+
"here is a contract anchor. The author's only repair is to "
|
|
22466
|
+
"reword a line that was correct."
|
|
22467
|
+
)
|
|
22468
|
+
for error in errors:
|
|
22469
|
+
report(f" {error}")
|
|
22470
|
+
return 1
|
|
22471
|
+
|
|
22472
|
+
# The control. A check that passes because nothing is refused any more
|
|
22473
|
+
# is indistinguishable from this one, so the inflections the owner
|
|
22474
|
+
# decided to keep are asserted in the same repository.
|
|
22475
|
+
for host in ("commit", "commits", "committed", "committing", "hashes"):
|
|
22476
|
+
state, errors = check(
|
|
22477
|
+
"- [x] A1 implementation\n"
|
|
22478
|
+
f" - Evidence:\n"
|
|
22479
|
+
f" - M1: pass — {host} a1b2c3d4e5f6 verified.\n"
|
|
22480
|
+
)
|
|
22481
|
+
if state == "unreported":
|
|
22482
|
+
report(f"{label}: keel --check reported no state at all on `{host}`.")
|
|
22483
|
+
return 1
|
|
22484
|
+
if state != "failed":
|
|
22485
|
+
report(
|
|
22486
|
+
f"{label}: an identifier beside `{host}` was accepted. That "
|
|
22487
|
+
"word names the act the rule exists to catch, and requiring "
|
|
22488
|
+
"a boundary must not stop it counting."
|
|
22489
|
+
)
|
|
22490
|
+
return 1
|
|
22491
|
+
if not [error for error in errors if "tasks.md:" in error]:
|
|
22492
|
+
report(f"{label}: the refusal on `{host}` named no line.")
|
|
22493
|
+
return 1
|
|
22494
|
+
|
|
22495
|
+
# The Chinese words carry no boundary, because none is definable
|
|
22496
|
+
# between two word characters. Asserted here rather than reasoned
|
|
22497
|
+
# about: `\b提交\b` matches none of these.
|
|
22498
|
+
for recorded in ("已提交", "未提交", "该任务尚未提交,等待评审"):
|
|
22499
|
+
state, _ = check(
|
|
22500
|
+
"- [x] A1 implementation\n"
|
|
22501
|
+
" - Evidence:\n"
|
|
22502
|
+
f" - M1: {recorded}。\n"
|
|
22503
|
+
)
|
|
22504
|
+
if state == "unreported":
|
|
22505
|
+
report(
|
|
22506
|
+
f"{label}: keel --check reported no state at all on "
|
|
22507
|
+
f"`{recorded}`. This is not a verdict about the fixture — "
|
|
22508
|
+
"the check did not reach the point of having one."
|
|
22509
|
+
)
|
|
22510
|
+
return 1
|
|
22511
|
+
if state != "failed":
|
|
22512
|
+
report(
|
|
22513
|
+
f"{label}: recorded state written as `{recorded}` was "
|
|
22514
|
+
"accepted. A word boundary around a Chinese context word "
|
|
22515
|
+
"disables it silently, which is why it does not carry one."
|
|
22516
|
+
)
|
|
22517
|
+
return 1
|
|
22518
|
+
|
|
22519
|
+
if label not in {name for name, _ in SCENARIOS}:
|
|
22520
|
+
report(f"{label}: the scenario registry does not include it.")
|
|
22521
|
+
return 1
|
|
22522
|
+
report(f"{label} scenario passed.")
|
|
22523
|
+
return 0
|
|
22524
|
+
|
|
22525
|
+
|
|
22526
|
+
def validate_a_covers_citation_is_not_a_record_scenario() -> int:
|
|
22527
|
+
"""Issue #65 §4: the rule refused a tasks.md that cited it.
|
|
22528
|
+
|
|
22529
|
+
A `Covers` entry names a requirement that must exist in a spec. When the
|
|
22530
|
+
requirement is about the state this check forbids — dirty worktrees,
|
|
22531
|
+
recorded identifiers — citing it by name reads to the check as recording
|
|
22532
|
+
it. The repair available to the author is to rename the requirement, which
|
|
22533
|
+
is what 5.19.0 did.
|
|
22534
|
+
|
|
22535
|
+
Eleven lines across four changes in this repository's history sit in this
|
|
22536
|
+
shape, eight of them citing requirements published in `openspec/specs/`.
|
|
22537
|
+
"""
|
|
22538
|
+
label = "a-covers-citation-is-not-a-record"
|
|
22539
|
+
# Two requirement names published in this repository today. Neither is
|
|
22540
|
+
# invented for the fixture; both are cited by archived changes.
|
|
22541
|
+
cited = (
|
|
22542
|
+
"keel-core-gates / Dirty-worktree attribution is conservative"
|
|
22543
|
+
" / A path already dirty at task start is not attributed",
|
|
22544
|
+
"keel-stateless-continuity / A recorded commit hash is recognized by"
|
|
22545
|
+
" what makes it one / A decimal run is not a hash",
|
|
22546
|
+
)
|
|
22547
|
+
|
|
22548
|
+
with tempfile.TemporaryDirectory(prefix="keel-covers-citation-") as raw:
|
|
22549
|
+
check, failure = _tasks_semantics_probe(raw)
|
|
22550
|
+
if failure is not None:
|
|
22551
|
+
report(f"{label}: keel --install failed while building the fixture.")
|
|
22552
|
+
report(failure)
|
|
22553
|
+
return 1
|
|
22554
|
+
|
|
22555
|
+
state, errors = check(
|
|
22556
|
+
"- [x] A1 implementation\n"
|
|
22557
|
+
" - Covers:\n"
|
|
22558
|
+
+ "".join(f" - {entry}\n" for entry in cited)
|
|
22559
|
+
+ " - Touch:\n - src/a.js\n"
|
|
22560
|
+
)
|
|
22561
|
+
if state == "unreported":
|
|
22562
|
+
report(f"{label}: keel --check reported no state at all on a citation.")
|
|
22563
|
+
return 1
|
|
22564
|
+
if state != "ok":
|
|
22565
|
+
report(
|
|
22566
|
+
f"{label}: a Covers citation was read as a record of the state "
|
|
22567
|
+
"it names. The entry is a reference to a requirement that has "
|
|
22568
|
+
"to exist elsewhere, and the only repair open to the author is "
|
|
22569
|
+
"to rename that requirement."
|
|
22570
|
+
)
|
|
22571
|
+
for error in errors:
|
|
22572
|
+
report(f" {error}")
|
|
22573
|
+
return 1
|
|
22574
|
+
|
|
22575
|
+
# The control, and the boundary. The same wording outside the field is
|
|
22576
|
+
# a statement, and is still refused — otherwise this scenario would
|
|
22577
|
+
# pass just as well against a check that had stopped running.
|
|
22578
|
+
state, errors = check(
|
|
22579
|
+
"- [x] A1 implementation\n"
|
|
22580
|
+
" - Covers:\n"
|
|
22581
|
+
+ "".join(f" - {entry}\n" for entry in cited)
|
|
22582
|
+
+ " - Evidence:\n"
|
|
22583
|
+
" - M1: pass — the worktree was dirty when the task started.\n"
|
|
22584
|
+
)
|
|
22585
|
+
if state == "unreported":
|
|
22586
|
+
report(
|
|
22587
|
+
f"{label}: keel --check reported no state at all while reading "
|
|
22588
|
+
"the wording outside the field."
|
|
22589
|
+
)
|
|
22590
|
+
return 1
|
|
22591
|
+
if state != "failed":
|
|
22592
|
+
report(
|
|
22593
|
+
f"{label}: the same wording in an Evidence line was accepted. "
|
|
22594
|
+
"The exemption is the Covers field, not the file that contains "
|
|
22595
|
+
"one."
|
|
22596
|
+
)
|
|
22597
|
+
return 1
|
|
22598
|
+
named = [error for error in errors if "tasks.md:" in error]
|
|
22599
|
+
if not named:
|
|
22600
|
+
report(f"{label}: the refusal outside the field named no line.")
|
|
22601
|
+
return 1
|
|
22602
|
+
# The citations are the two lines under the label, and each carries
|
|
22603
|
+
# wording this rule refuses; naming either of them means the exemption
|
|
22604
|
+
# did not hold while the Evidence line was being refused.
|
|
22605
|
+
if any(f"tasks.md:{cited_line}:" in error
|
|
22606
|
+
for error in named for cited_line in (11, 12)):
|
|
22607
|
+
report(
|
|
22608
|
+
f"{label}: the refusal named a citation line. The rule fired "
|
|
22609
|
+
"correctly on the Evidence line and wrongly on the field, so "
|
|
22610
|
+
"what is wrong is the field's bound."
|
|
22611
|
+
)
|
|
22612
|
+
for error in named:
|
|
22613
|
+
report(f" {error}")
|
|
22614
|
+
return 1
|
|
22615
|
+
|
|
22616
|
+
# The bound ends where the next field begins. A `Covers` field that
|
|
22617
|
+
# ran to the end of the task would swallow the Evidence line above and
|
|
22618
|
+
# make that control vacuous, so the end of the field is asserted from
|
|
22619
|
+
# the other side too.
|
|
22620
|
+
state, _ = check(
|
|
22621
|
+
"- [x] A1 implementation\n"
|
|
22622
|
+
" - Covers:\n"
|
|
22623
|
+
f" - {cited[0]}\n"
|
|
22624
|
+
" - Verify:\n"
|
|
22625
|
+
" - M1: the change is 已提交 and needs no further work.\n"
|
|
22626
|
+
)
|
|
22627
|
+
if state == "unreported":
|
|
22628
|
+
report(
|
|
22629
|
+
f"{label}: keel --check reported no state at all while reading "
|
|
22630
|
+
"the field that follows a Covers field."
|
|
22631
|
+
)
|
|
22632
|
+
return 1
|
|
22633
|
+
if state != "failed":
|
|
22634
|
+
report(
|
|
22635
|
+
f"{label}: a Verify line following a Covers field was exempted. "
|
|
22636
|
+
"The field ends at the next field label."
|
|
22637
|
+
)
|
|
22638
|
+
return 1
|
|
22639
|
+
|
|
22640
|
+
if label not in {name for name, _ in SCENARIOS}:
|
|
22641
|
+
report(f"{label}: the scenario registry does not include it.")
|
|
22642
|
+
return 1
|
|
22643
|
+
report(f"{label} scenario passed.")
|
|
22644
|
+
return 0
|
|
22645
|
+
|
|
22646
|
+
|
|
22647
|
+
def validate_validation_runner_scenario() -> int:
|
|
22648
|
+
if "SCENARIOS" not in globals():
|
|
22649
|
+
report("validation-runner: the scenario registry is missing.")
|
|
22650
|
+
return 1
|
|
19351
22651
|
names = [name for name, _ in SCENARIOS]
|
|
19352
22652
|
if len(names) != len(set(names)):
|
|
19353
22653
|
report("validation-runner: registry names are not unique.")
|
|
@@ -19482,6 +22782,200 @@ def validate_doctor_openspec_honesty_scenario() -> int:
|
|
|
19482
22782
|
return 0
|
|
19483
22783
|
|
|
19484
22784
|
|
|
22785
|
+
# A scenario name, as the registry spells one. Two registered names carry no
|
|
22786
|
+
# hyphen — `cli` and `uninstall` — so requiring one would leave exactly those
|
|
22787
|
+
# two unchecked, and allowing single words was measured to add no false
|
|
22788
|
+
# positive.
|
|
22789
|
+
AUTHORED_SCENARIO_NAME = re.compile(r"^[a-z0-9][a-z0-9-]*$")
|
|
22790
|
+
|
|
22791
|
+
# The two forms a scenario reference is read in. `--scenario` is the invocation
|
|
22792
|
+
# itself and needs no context. The other is keyed on the *assertion* rather than
|
|
22793
|
+
# on the token, because kebab-case is this project's spelling for gate stages,
|
|
22794
|
+
# diagnostic codes, skills, capabilities, and hook events too: reading every
|
|
22795
|
+
# such token as a scenario name flags 22 of the 55 in the archive. Keying on
|
|
22796
|
+
# "stays green" flags one, and that one is a real unregistered name.
|
|
22797
|
+
AUTHORED_SCENARIO_FLAG = re.compile(r"--scenario\s+`?([A-Za-z0-9][\w.-]*)`?")
|
|
22798
|
+
AUTHORED_SCENARIO_STAYS_GREEN = re.compile(r"stays? green", re.IGNORECASE)
|
|
22799
|
+
AUTHORED_CHECK_LINE = re.compile(r"^-\s*M\d+\b")
|
|
22800
|
+
AUTHORED_VERIFY_BLOCK = re.compile(r"^-\s*(?:Verify|Commands):\s*$")
|
|
22801
|
+
AUTHORED_CAPSULE_BLOCK = re.compile(r"^-\s*[A-Z][A-Za-z ]*:\s*$")
|
|
22802
|
+
AUTHORED_INLINE_CODE = re.compile(r"`([^`\n]+)`")
|
|
22803
|
+
|
|
22804
|
+
AUTHORED_SCENARIO_FORMS = (
|
|
22805
|
+
"A scenario reference is read in two forms: a name after `--scenario`, and "
|
|
22806
|
+
"a backticked lowercase token in a `Verify` M<n> check asserting that "
|
|
22807
|
+
"something stays green. Other backticked vocabulary in a check is not read "
|
|
22808
|
+
"as a scenario name."
|
|
22809
|
+
)
|
|
22810
|
+
|
|
22811
|
+
|
|
22812
|
+
def unregistered_scenario_references(
|
|
22813
|
+
text: str, registered: set[str]
|
|
22814
|
+
) -> list[tuple[int, str, str]]:
|
|
22815
|
+
"""Scenario names a task contract declares that the registry does not hold.
|
|
22816
|
+
|
|
22817
|
+
Returns `(line, name, form)` for each, 1-indexed, in file order. The `form`
|
|
22818
|
+
is which of the two recognized shapes matched, so a report can say why the
|
|
22819
|
+
token was read as a scenario name at all.
|
|
22820
|
+
"""
|
|
22821
|
+
problems: list[tuple[int, str, str]] = []
|
|
22822
|
+
in_verify = False
|
|
22823
|
+
for line_number, line in enumerate(text.splitlines(), start=1):
|
|
22824
|
+
stripped = line.strip()
|
|
22825
|
+
# A capsule block heading ends the previous block, so `Evidence` and
|
|
22826
|
+
# `Stop Rules` leave the Verify block rather than extending it. Evidence
|
|
22827
|
+
# is written after the run, when execution has already answered the
|
|
22828
|
+
# question this check exists to answer earlier.
|
|
22829
|
+
if AUTHORED_CAPSULE_BLOCK.match(stripped):
|
|
22830
|
+
in_verify = bool(AUTHORED_VERIFY_BLOCK.match(stripped))
|
|
22831
|
+
for match in AUTHORED_SCENARIO_FLAG.finditer(line):
|
|
22832
|
+
name = match.group(1)
|
|
22833
|
+
if name not in registered:
|
|
22834
|
+
problems.append((line_number, name, "--scenario"))
|
|
22835
|
+
if not in_verify or not AUTHORED_CHECK_LINE.match(stripped):
|
|
22836
|
+
continue
|
|
22837
|
+
if not AUTHORED_SCENARIO_STAYS_GREEN.search(line):
|
|
22838
|
+
continue
|
|
22839
|
+
for token in AUTHORED_INLINE_CODE.findall(line):
|
|
22840
|
+
if not AUTHORED_SCENARIO_NAME.match(token):
|
|
22841
|
+
continue
|
|
22842
|
+
if token not in registered:
|
|
22843
|
+
problems.append((line_number, token, "stays green"))
|
|
22844
|
+
return problems
|
|
22845
|
+
|
|
22846
|
+
|
|
22847
|
+
def active_change_tasks(repo: Path) -> list[Path]:
|
|
22848
|
+
"""Every active change's tasks.md. Archived changes are deliberately absent.
|
|
22849
|
+
|
|
22850
|
+
An archived task records what was true when it ran; a scenario renamed
|
|
22851
|
+
afterwards does not make that record wrong, and turning history red for it
|
|
22852
|
+
tells a reader nothing they can act on.
|
|
22853
|
+
"""
|
|
22854
|
+
changes = repo / "openspec" / "changes"
|
|
22855
|
+
archive = changes / "archive"
|
|
22856
|
+
# The walk finds archived changes and the filter is what drops them, rather
|
|
22857
|
+
# than a glob depth that happens not to reach them. A depth that excludes by
|
|
22858
|
+
# accident passes every test aimed at the exclusion, including one aimed at
|
|
22859
|
+
# it on purpose.
|
|
22860
|
+
return sorted(
|
|
22861
|
+
path for path in changes.rglob("tasks.md") if archive not in path.parents
|
|
22862
|
+
)
|
|
22863
|
+
|
|
22864
|
+
|
|
22865
|
+
def validate_authored_scenario_names_scenario() -> int:
|
|
22866
|
+
"""Issue #51: a check whose scenario name is not registered cannot run.
|
|
22867
|
+
|
|
22868
|
+
Three of the four authored contracts #51 records named something that did
|
|
22869
|
+
not exist, two of them scenario names, both written as an `M5 (regression)`
|
|
22870
|
+
check asserting that named scenarios stay green, and both found only when
|
|
22871
|
+
the check was executed. A fifth, unrecorded there, is
|
|
22872
|
+
`2026-08-01-the-name-is-not-the-thing/tasks.md:132`, which declares
|
|
22873
|
+
`tasks-template-validates` — a name `git log -S` finds at no commit — and
|
|
22874
|
+
records that check as passing.
|
|
22875
|
+
|
|
22876
|
+
The assertions live on synthetic content with a known name set, because the
|
|
22877
|
+
live scan reads whatever the working tree holds and is normally empty: a
|
|
22878
|
+
change is archived when it closes. The temp-repository half is the positive
|
|
22879
|
+
control for the whole path, so a run that passes because the file selection
|
|
22880
|
+
silently found nothing is distinguishable from one that verified something.
|
|
22881
|
+
"""
|
|
22882
|
+
registered = {name for name, _ in SCENARIOS}
|
|
22883
|
+
known = {"core-gates", "cli"}
|
|
22884
|
+
authored = (
|
|
22885
|
+
"# Tasks\n"
|
|
22886
|
+
"\n"
|
|
22887
|
+
"- [ ] 1.1 Do the thing\n"
|
|
22888
|
+
" - Touch:\n"
|
|
22889
|
+
" - src/thing.js\n"
|
|
22890
|
+
" - Verify:\n"
|
|
22891
|
+
" - Strategy: vertical-tdd\n"
|
|
22892
|
+
" - M1: `node scripts/run_python.js scripts/validate_plugin.py "
|
|
22893
|
+
"--scenario gate-diagnostics` passes\n"
|
|
22894
|
+
" - M2 (regression): `core-gates` and `target-surface-doctor` stay "
|
|
22895
|
+
"green\n"
|
|
22896
|
+
" - M3 (regression): `task-start`, `keel-review-checklist`, "
|
|
22897
|
+
"`contract-drift`, `subagent-stop`, and `keel-task-capsule` are "
|
|
22898
|
+
"unchanged\n"
|
|
22899
|
+
" - M4: `cli` and "
|
|
22900
|
+
"`node scripts/run_python.js scripts/validate_plugin.py "
|
|
22901
|
+
"--scenario core-gates` stay green\n"
|
|
22902
|
+
" - Evidence:\n"
|
|
22903
|
+
" - M2: `target-surface-doctor` stays green\n"
|
|
22904
|
+
)
|
|
22905
|
+
found = unregistered_scenario_references(authored, known)
|
|
22906
|
+
expected = [
|
|
22907
|
+
(8, "gate-diagnostics", "--scenario"),
|
|
22908
|
+
(9, "target-surface-doctor", "stays green"),
|
|
22909
|
+
]
|
|
22910
|
+
if found != expected:
|
|
22911
|
+
report(
|
|
22912
|
+
"authored-scenario-names-are-registered: the recognized forms did "
|
|
22913
|
+
f"not report what they must. expected {expected}, found {found}."
|
|
22914
|
+
)
|
|
22915
|
+
report(AUTHORED_SCENARIO_FORMS)
|
|
22916
|
+
return 1
|
|
22917
|
+
|
|
22918
|
+
# Which files are read is a separate question from what is read out of
|
|
22919
|
+
# them, and it is the half that can silently resolve to nothing. The
|
|
22920
|
+
# planted reference is the positive control for the whole path: an
|
|
22921
|
+
# extractor that stopped reporting, or a selection that stopped finding,
|
|
22922
|
+
# both fail here rather than passing on an empty scan.
|
|
22923
|
+
with tempfile.TemporaryDirectory(prefix="keel-authored-names-") as raw:
|
|
22924
|
+
repo = Path(raw)
|
|
22925
|
+
write_text(repo / "openspec/changes/live/tasks.md", authored)
|
|
22926
|
+
write_text(repo / "openspec/changes/archive/old/tasks.md", authored)
|
|
22927
|
+
selected = active_change_tasks(repo)
|
|
22928
|
+
if selected != [repo / "openspec/changes/live/tasks.md"]:
|
|
22929
|
+
report(
|
|
22930
|
+
"authored-scenario-names-are-registered: the active change "
|
|
22931
|
+
"selection is wrong. An archived tasks.md must not be scanned "
|
|
22932
|
+
"and an active one must be. "
|
|
22933
|
+
f"selected {[str(path) for path in selected]}."
|
|
22934
|
+
)
|
|
22935
|
+
return 1
|
|
22936
|
+
control = [
|
|
22937
|
+
(path, unregistered_scenario_references(
|
|
22938
|
+
path.read_text(encoding="utf-8"), known
|
|
22939
|
+
))
|
|
22940
|
+
for path in selected
|
|
22941
|
+
]
|
|
22942
|
+
if [problems for _, problems in control] != [expected]:
|
|
22943
|
+
report(
|
|
22944
|
+
"authored-scenario-names-are-registered: a planted reference "
|
|
22945
|
+
"in an active change was not reported end to end, so a clean "
|
|
22946
|
+
"run over the real tree would prove nothing. "
|
|
22947
|
+
f"found {control}."
|
|
22948
|
+
)
|
|
22949
|
+
return 1
|
|
22950
|
+
|
|
22951
|
+
for path in active_change_tasks(ROOT):
|
|
22952
|
+
problems = unregistered_scenario_references(
|
|
22953
|
+
path.read_text(encoding="utf-8"), registered
|
|
22954
|
+
)
|
|
22955
|
+
if not problems:
|
|
22956
|
+
continue
|
|
22957
|
+
relative = path.relative_to(ROOT).as_posix()
|
|
22958
|
+
for line_number, name, form in problems:
|
|
22959
|
+
report(
|
|
22960
|
+
f"authored-scenario-names-are-registered: {relative}:"
|
|
22961
|
+
f"{line_number} names `{name}` after {form}, and no scenario by "
|
|
22962
|
+
"that name is registered, so the check that names it cannot run."
|
|
22963
|
+
)
|
|
22964
|
+
report(AUTHORED_SCENARIO_FORMS)
|
|
22965
|
+
return 1
|
|
22966
|
+
|
|
22967
|
+
if "authored-scenario-names-are-registered" not in {
|
|
22968
|
+
name for name, _ in SCENARIOS
|
|
22969
|
+
}:
|
|
22970
|
+
report(
|
|
22971
|
+
"authored-scenario-names-are-registered: the scenario registry "
|
|
22972
|
+
"does not include it."
|
|
22973
|
+
)
|
|
22974
|
+
return 1
|
|
22975
|
+
report("authored-scenario-names-are-registered scenario passed.")
|
|
22976
|
+
return 0
|
|
22977
|
+
|
|
22978
|
+
|
|
19485
22979
|
SCENARIOS: tuple = (
|
|
19486
22980
|
("stateless-continuity", validate_stateless_continuity_scenario),
|
|
19487
22981
|
("core-gates", validate_core_gates_scenario),
|
|
@@ -19491,6 +22985,10 @@ SCENARIOS: tuple = (
|
|
|
19491
22985
|
("target-surface", validate_target_surface_scenario),
|
|
19492
22986
|
("thin-native-install", validate_thin_native_install_scenario),
|
|
19493
22987
|
("expectation-slice-gates", validate_expectation_slice_gates_scenario),
|
|
22988
|
+
(
|
|
22989
|
+
"unparsed-covers-critical-statement",
|
|
22990
|
+
validate_unparsed_covers_critical_statement_scenario,
|
|
22991
|
+
),
|
|
19494
22992
|
("expectation-completion-gates", validate_expectation_completion_gates_scenario),
|
|
19495
22993
|
("authoring-continuity", validate_authoring_continuity_scenario),
|
|
19496
22994
|
("domain-lenses", validate_domain_lenses_scenario),
|
|
@@ -19499,8 +22997,16 @@ SCENARIOS: tuple = (
|
|
|
19499
22997
|
("sync-surface-overlay", validate_sync_surface_overlay_scenario),
|
|
19500
22998
|
("openspec-surface-overlay", validate_openspec_surface_overlay_scenario),
|
|
19501
22999
|
("uninstall", validate_uninstall_scenario),
|
|
23000
|
+
(
|
|
23001
|
+
"uninstall-removes-the-overlay",
|
|
23002
|
+
validate_uninstall_removes_the_overlay_scenario,
|
|
23003
|
+
),
|
|
19502
23004
|
("cli", validate_cli_scenario),
|
|
19503
23005
|
("doctor-openspec-honesty", validate_doctor_openspec_honesty_scenario),
|
|
23006
|
+
(
|
|
23007
|
+
"authored-scenario-names-are-registered",
|
|
23008
|
+
validate_authored_scenario_names_scenario,
|
|
23009
|
+
),
|
|
19504
23010
|
("update-pack-install", validate_update_pack_install_scenario),
|
|
19505
23011
|
("update-default-registry", validate_update_default_registry_scenario),
|
|
19506
23012
|
("verification-layering-docs", validate_verification_layering_docs_scenario),
|
|
@@ -19508,6 +23014,10 @@ SCENARIOS: tuple = (
|
|
|
19508
23014
|
"standing-authorization-declaration",
|
|
19509
23015
|
validate_standing_authorization_declaration_scenario,
|
|
19510
23016
|
),
|
|
23017
|
+
(
|
|
23018
|
+
"standing-authorization-sync-confusion",
|
|
23019
|
+
validate_standing_authorization_sync_confusion_scenario,
|
|
23020
|
+
),
|
|
19511
23021
|
(
|
|
19512
23022
|
"standing-authorization-inheritance",
|
|
19513
23023
|
validate_standing_authorization_inheritance_scenario,
|
|
@@ -19523,6 +23033,10 @@ SCENARIOS: tuple = (
|
|
|
19523
23033
|
("precedent-never-weakens", validate_precedent_never_weakens_scenario),
|
|
19524
23034
|
("precedent-rules", validate_precedent_rules_scenario),
|
|
19525
23035
|
("triage-declaration", validate_triage_declaration_scenario),
|
|
23036
|
+
(
|
|
23037
|
+
"triage-admits-from-the-repository",
|
|
23038
|
+
validate_triage_admits_from_the_repository_scenario,
|
|
23039
|
+
),
|
|
19526
23040
|
("delegation-declaration", validate_delegation_declaration_scenario),
|
|
19527
23041
|
("delegation-inheritance", validate_delegation_inheritance_scenario),
|
|
19528
23042
|
("native-capability-scope", validate_native_capability_scope_scenario),
|
|
@@ -19553,6 +23067,10 @@ SCENARIOS: tuple = (
|
|
|
19553
23067
|
),
|
|
19554
23068
|
("inline-code-is-concrete", validate_inline_code_is_concrete_scenario),
|
|
19555
23069
|
("covers-separator-collision", validate_covers_separator_collision_scenario),
|
|
23070
|
+
(
|
|
23071
|
+
"unresolved-covers-names-what-failed",
|
|
23072
|
+
validate_unresolved_covers_names_what_failed_scenario,
|
|
23073
|
+
),
|
|
19556
23074
|
(
|
|
19557
23075
|
"unresolved-authority-names-field",
|
|
19558
23076
|
validate_unresolved_authority_names_field_scenario,
|
|
@@ -19565,6 +23083,10 @@ SCENARIOS: tuple = (
|
|
|
19565
23083
|
"non-concrete-check-names-token",
|
|
19566
23084
|
validate_non_concrete_check_names_token_scenario,
|
|
19567
23085
|
),
|
|
23086
|
+
(
|
|
23087
|
+
"unusable-contract-names-only-its-cause",
|
|
23088
|
+
validate_unusable_contract_names_only_its_cause_scenario,
|
|
23089
|
+
),
|
|
19568
23090
|
(
|
|
19569
23091
|
"absent-verification-form-is-one-problem",
|
|
19570
23092
|
validate_absent_verification_form_is_one_problem_scenario,
|
|
@@ -19625,12 +23147,20 @@ SCENARIOS: tuple = (
|
|
|
19625
23147
|
("tracker-durable-owner", validate_tracker_durable_owner_scenario),
|
|
19626
23148
|
("findings-resolved-here", validate_findings_resolved_here_scenario),
|
|
19627
23149
|
("guard-stale-manifest", validate_guard_stale_manifest_scenario),
|
|
23150
|
+
(
|
|
23151
|
+
"guard-status-stale-manifest",
|
|
23152
|
+
validate_guard_status_stale_manifest_scenario,
|
|
23153
|
+
),
|
|
19628
23154
|
("assertion-shape-count", validate_assertion_shape_count_scenario),
|
|
19629
23155
|
("guard-manifest-ignored", validate_guard_manifest_ignored_scenario),
|
|
19630
23156
|
(
|
|
19631
23157
|
"guard-status-is-not-enforcement",
|
|
19632
23158
|
validate_guard_status_is_not_enforcement_scenario,
|
|
19633
23159
|
),
|
|
23160
|
+
(
|
|
23161
|
+
"guard-warnings-are-concise",
|
|
23162
|
+
validate_guard_warnings_are_concise_scenario,
|
|
23163
|
+
),
|
|
19634
23164
|
("source-repo-cli-resolution", validate_source_repo_cli_resolution_scenario),
|
|
19635
23165
|
("task-contract-core", validate_task_contract_core_scenario),
|
|
19636
23166
|
("task-capsule", validate_task_capsule_scenario),
|
|
@@ -19694,7 +23224,31 @@ SCENARIOS: tuple = (
|
|
|
19694
23224
|
("domain-lens-doctor", validate_domain_lens_doctor_scenario),
|
|
19695
23225
|
("plan-funnel-guidance", validate_plan_funnel_guidance_scenario),
|
|
19696
23226
|
("native-tasks-view", validate_native_tasks_view_scenario),
|
|
23227
|
+
(
|
|
23228
|
+
"declared-paths-are-read-whole",
|
|
23229
|
+
validate_declared_paths_are_read_whole_scenario,
|
|
23230
|
+
),
|
|
23231
|
+
(
|
|
23232
|
+
"published-specs-validate-strictly",
|
|
23233
|
+
validate_published_specs_validate_strictly_scenario,
|
|
23234
|
+
),
|
|
23235
|
+
(
|
|
23236
|
+
"decimal-runs-are-not-hash-shaped",
|
|
23237
|
+
validate_decimal_runs_are_not_hash_shaped_scenario,
|
|
23238
|
+
),
|
|
23239
|
+
("a-context-word-is-a-word", validate_a_context_word_is_a_word_scenario),
|
|
23240
|
+
(
|
|
23241
|
+
"a-covers-citation-is-not-a-record",
|
|
23242
|
+
validate_a_covers_citation_is_not_a_record_scenario,
|
|
23243
|
+
),
|
|
19697
23244
|
("validation-runner", validate_validation_runner_scenario),
|
|
23245
|
+
("section-boundary", validate_section_boundary_scenario),
|
|
23246
|
+
("review-entry-extent", validate_review_entry_extent_scenario),
|
|
23247
|
+
("reauthorizations-shape", validate_reauthorizations_shape_scenario),
|
|
23248
|
+
(
|
|
23249
|
+
"change-verify-deferred-evidence",
|
|
23250
|
+
validate_change_verify_deferred_evidence_scenario,
|
|
23251
|
+
),
|
|
19698
23252
|
)
|
|
19699
23253
|
|
|
19700
23254
|
|