@agent-ix/quoin 0.12.0 → 0.12.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{base-DGA47VMM.js → base-DD0ONCEf.js} +1 -1
- package/dist/cli.d.ts +15 -0
- package/dist/cli.d.ts.map +1 -1
- package/dist/cli.js +11 -7
- package/dist/commands/catalog/list.js +2 -2
- package/dist/commands/catalog/show.js +2 -2
- package/dist/commands/catalog/validate.js +2 -2
- package/dist/commands/config/doctor.js +1 -1
- package/dist/commands/config/edit.js +1 -1
- package/dist/commands/config/get.js +1 -1
- package/dist/commands/config/set.js +1 -1
- package/dist/commands/matrix.js +1 -1
- package/dist/commands/module/ensure-defaults.js +3 -3
- package/dist/commands/module/install.js +2 -2
- package/dist/commands/module/list.js +2 -2
- package/dist/commands/module/remove.js +2 -2
- package/dist/commands/review.js +1 -1
- package/dist/commands/to-plan.js +1 -1
- package/dist/commands/update.js +2 -2
- package/dist/commands/write.js +3 -3
- package/dist/{flow-command-CPS23AjI.js → flow-command-DK5LDrtg.js} +1 -1
- package/dist/hooks/command-not-found.d.ts +23 -0
- package/dist/hooks/command-not-found.d.ts.map +1 -0
- package/dist/hooks/command-not-found.js +8 -0
- package/dist/index.js +6 -6
- package/dist/{modules-CCvL41O_.js → modules-_4_rnXpw.js} +2 -2
- package/dist/{plugins-ByvihMa0.js → plugins-BO1uLsfi.js} +1 -1
- package/dist/{version-Be_NtSa4.js → version-DIJF4enj.js} +1 -1
- package/dist/{write-xs9kemgB.js → write-D462q_kk.js} +1 -1
- package/package.json +5 -2
- package/skills/gap-analysis/SKILL.md +5 -3
- package/skills/gap-analysis/references/step-3-matrix-verification.md +94 -24
- package/skills/gap-analysis/references/step-6-specreview-artifact.md +6 -1
- package/skills/spec-correctness/SKILL.md +11 -13
- package/skills/spec-correctness/assets/strategy-table.md +3 -3
- package/skills/spec-correctness/assets/templates/fast-check.ts.md +41 -25
- package/skills/spec-correctness/assets/templates/hypothesis.py.md +16 -15
- package/skills/spec-correctness/assets/templates/proptest.rs.md +12 -9
- package/skills/spec-correctness/references/step-0-scope-and-harness.md +6 -6
- package/skills/spec-correctness/references/step-1-census.md +1 -1
- package/skills/spec-correctness/references/step-2-clause-grounding.md +2 -2
- package/skills/spec-correctness/references/step-3-strategy-selection.md +13 -11
- package/skills/spec-correctness/references/step-4-generate-tests.md +43 -25
- package/skills/spec-correctness/references/step-5-second-pass.md +11 -9
- package/skills/spec-correctness/references/step-6-review-artifact.md +121 -0
- package/skills/spec-correctness/references/step-7-report-and-handoff.md +38 -17
- package/skills/spec-correctness/assets/property-queue-template.md +0 -78
- package/skills/spec-correctness/references/step-6-review-queue.md +0 -64
|
@@ -1,7 +1,6 @@
|
|
|
1
1
|
# TypeScript / fast-check templates
|
|
2
2
|
|
|
3
|
-
File: `tests/props/fr-NNN.prop.test.ts
|
|
4
|
-
Inert marker: `it.skip` / `describe.skip`.
|
|
3
|
+
File: `tests/props/fr-NNN.prop.test.ts`.
|
|
5
4
|
Runner: whichever the repo already uses — read `vitest.config.*`, `jest.config.*`, or
|
|
6
5
|
`package.json#scripts.test`. The imports below assume vitest; swap for `@jest/globals`
|
|
7
6
|
where the repo uses jest.
|
|
@@ -20,11 +19,11 @@ const entryArb: fc.Arbitrary<Entry> = fc.record({
|
|
|
20
19
|
name: fc.stringMatching(/^[a-z]{1,12}$/),
|
|
21
20
|
});
|
|
22
21
|
|
|
23
|
-
/**
|
|
24
|
-
* Trace: FR-012-AC-1 — declaring modules are listed in sorted order.
|
|
25
|
-
* spec-correctness: row=FR-012-AC-1 property=ordering extraction=extractable origin=regex review=none
|
|
26
|
-
*/
|
|
27
22
|
describe("FR-012-AC-1 duplicate modules are sorted", () => {
|
|
23
|
+
/**
|
|
24
|
+
* Trace: FR-012-AC-1 — declaring modules are listed in sorted order.
|
|
25
|
+
* spec-correctness: row=FR-012-AC-1 property=ordering extraction=extractable origin=regex review=none
|
|
26
|
+
*/
|
|
28
27
|
it("holds for any catalog", () => {
|
|
29
28
|
fc.assert(
|
|
30
29
|
fc.property(fc.array(entryArb, { maxLength: 32 }), (entries) => {
|
|
@@ -40,6 +39,18 @@ describe("FR-012-AC-1 duplicate modules are sorted", () => {
|
|
|
40
39
|
Both tag carriers: `Trace:` on its own JSDoc line, and `FR-012-AC-1` inside the `describe`
|
|
41
40
|
title.
|
|
42
41
|
|
|
42
|
+
**The `Trace:` block goes immediately above the `it(` / `test(` registration — never above
|
|
43
|
+
the enclosing `describe`.** A tag binds to the symbol whose source span encloses it, and
|
|
44
|
+
`describe(…)` groups tests but *registers none itself* (quire-rs
|
|
45
|
+
`src/symbols/typescript.rs`, `registration`). A tag above a `describe` therefore attaches
|
|
46
|
+
to nothing: it reads fine, greps fine, and is invisible to `quire coverage`. That is the
|
|
47
|
+
whole of agent-ix/quoin#61 — measured, one tag moved from its `describe` to its `test(`
|
|
48
|
+
took `FR-005` from 0/3 to 1/3 with no other change.
|
|
49
|
+
|
|
50
|
+
Python and Rust are unaffected: a symbol's span covers its annotation block, declaration
|
|
51
|
+
**and body**, so a docstring inside the function and an attribute above `#[test] fn` both
|
|
52
|
+
land inside a real test symbol. Only the TS placement can fall outside one.
|
|
53
|
+
|
|
43
54
|
## Family bodies
|
|
44
55
|
|
|
45
56
|
```ts
|
|
@@ -63,11 +74,11 @@ expect(summarize(xs)).toEqual(summarize([...xs].reverse()));
|
|
|
63
74
|
## error-case — negative domain
|
|
64
75
|
|
|
65
76
|
```ts
|
|
66
|
-
/**
|
|
67
|
-
* Trace: FR-005-AC-1 — an unknown command raises an error that names the usage.
|
|
68
|
-
* spec-correctness: row=FR-005-AC-1 property=error-case extraction=extractable origin=regex review=none
|
|
69
|
-
*/
|
|
70
77
|
describe("FR-005-AC-1 unknown commands error", () => {
|
|
78
|
+
/**
|
|
79
|
+
* Trace: FR-005-AC-1 — an unknown command raises an error that names the usage.
|
|
80
|
+
* spec-correctness: row=FR-005-AC-1 property=error-case extraction=extractable origin=regex review=none
|
|
81
|
+
*/
|
|
71
82
|
it("rejects any command outside the known set", () => {
|
|
72
83
|
fc.assert(
|
|
73
84
|
fc.property(fc.stringMatching(/^[a-z]{1,12}$/), (cmd) => {
|
|
@@ -83,11 +94,11 @@ describe("FR-005-AC-1 unknown commands error", () => {
|
|
|
83
94
|
## lifecycle — fc.commands / modelRun
|
|
84
95
|
|
|
85
96
|
```ts
|
|
86
|
-
/**
|
|
87
|
-
* Trace: FR-040-AC-2 — a session accepts exactly the ops its state allows.
|
|
88
|
-
* spec-correctness: row=FR-040-AC-2 property=lifecycle extraction=extractable origin=regex review=none
|
|
89
|
-
*/
|
|
90
97
|
describe("FR-040-AC-2 session lifecycle", () => {
|
|
98
|
+
/**
|
|
99
|
+
* Trace: FR-040-AC-2 — a session accepts exactly the ops its state allows.
|
|
100
|
+
* spec-correctness: row=FR-040-AC-2 property=lifecycle extraction=extractable origin=regex review=none
|
|
101
|
+
*/
|
|
91
102
|
it("matches the model over any op sequence", () => {
|
|
92
103
|
fc.assert(
|
|
93
104
|
fc.property(fc.commands(allCommands, { maxCommands: 24 }), (cmds) =>
|
|
@@ -104,11 +115,11 @@ assertion. That split *is* the precondition/oracle split from step 2.
|
|
|
104
115
|
## concurrency — deterministic scheduler
|
|
105
116
|
|
|
106
117
|
```ts
|
|
107
|
-
/**
|
|
108
|
-
* Trace: FR-050-AC-1 — concurrent writers never lose an entry.
|
|
109
|
-
* spec-correctness: row=FR-050-AC-1 property=concurrency extraction=extractable origin=regex review=none
|
|
110
|
-
*/
|
|
111
118
|
describe("FR-050-AC-1 concurrent writes", () => {
|
|
119
|
+
/**
|
|
120
|
+
* Trace: FR-050-AC-1 — concurrent writers never lose an entry.
|
|
121
|
+
* spec-correctness: row=FR-050-AC-1 property=concurrency extraction=extractable origin=regex review=none
|
|
122
|
+
*/
|
|
112
123
|
it("linearizes under any interleaving", async () => {
|
|
113
124
|
await fc.assert(
|
|
114
125
|
fc.asyncProperty(fc.scheduler(), async (s) => {
|
|
@@ -128,19 +139,24 @@ describe("FR-050-AC-1 concurrent writes", () => {
|
|
|
128
139
|
`fc.scheduler()` is deterministic and shrinkable — never use real timers or `Promise.all`
|
|
129
140
|
races for a concurrency property.
|
|
130
141
|
|
|
131
|
-
##
|
|
142
|
+
## A `candidate` record
|
|
143
|
+
|
|
144
|
+
Same file, same path, and it **runs**. `extraction: candidate` means the metamorphic label
|
|
145
|
+
was not corroborated structurally — something for a reviewer to read, not a reason to
|
|
146
|
+
disable a test:
|
|
132
147
|
|
|
133
148
|
```ts
|
|
134
|
-
/**
|
|
135
|
-
* Trace: FR-018-AC-3 — a plugin source maps to exactly one resolved root.
|
|
136
|
-
* spec-correctness: row=FR-018-AC-3 property=invariant extraction=candidate origin=regex-candidate review=required
|
|
137
|
-
*/
|
|
138
149
|
describe("FR-018-AC-3 source maps to one root", () => {
|
|
139
|
-
|
|
150
|
+
/**
|
|
151
|
+
* Trace: FR-018-AC-3 — a plugin source maps to exactly one resolved root.
|
|
152
|
+
* spec-correctness: row=FR-018-AC-3 property=invariant extraction=candidate origin=regex-candidate
|
|
153
|
+
*/
|
|
154
|
+
it("holds for any plugin source", () => { … });
|
|
140
155
|
});
|
|
141
156
|
```
|
|
142
157
|
|
|
143
|
-
|
|
158
|
+
What marks it for review is a `medium` finding in the review artifact (step 6), not an
|
|
159
|
+
`it.skip` in the tree.
|
|
144
160
|
|
|
145
161
|
## Notes
|
|
146
162
|
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
# Python / hypothesis templates
|
|
2
2
|
|
|
3
|
-
File: `tests/props/test_fr_NNN.py
|
|
4
|
-
|
|
3
|
+
File: `tests/props/test_fr_NNN.py`.
|
|
4
|
+
Every emitted test runs — there is no inert form.
|
|
5
5
|
|
|
6
6
|
## Anatomy
|
|
7
7
|
|
|
@@ -113,37 +113,38 @@ TestSessionLifecycle = SessionMachine.TestCase
|
|
|
113
113
|
|
|
114
114
|
`@precondition` is the step-2 precondition; `@invariant` is the step-2 oracle.
|
|
115
115
|
|
|
116
|
-
## concurrency — always
|
|
116
|
+
## concurrency — always a downgrade
|
|
117
117
|
|
|
118
|
-
hypothesis ships no interleaving scheduler, so a Python concurrency criterion
|
|
119
|
-
|
|
118
|
+
hypothesis ships no interleaving scheduler, so a Python concurrency criterion never gets
|
|
119
|
+
real interleaving coverage. Emit a `RuleBasedStateMachine` over the async model, and record
|
|
120
|
+
the downgrade as a finding:
|
|
120
121
|
|
|
121
122
|
```python
|
|
122
|
-
@pytest.mark.skip(reason="spec-correctness review pending")
|
|
123
123
|
class TestConcurrentWrites(RuleBasedStateMachine):
|
|
124
124
|
"""Trace: FR-050-AC-1 — concurrent writers never lose an entry.
|
|
125
125
|
|
|
126
|
-
spec-correctness: row=FR-050-AC-1 property=concurrency extraction=not-extractable origin=llm-second-pass
|
|
126
|
+
spec-correctness: row=FR-050-AC-1 property=concurrency extraction=not-extractable origin=llm-second-pass confidence=low
|
|
127
127
|
"""
|
|
128
128
|
```
|
|
129
129
|
|
|
130
|
-
|
|
131
|
-
|
|
130
|
+
The finding says the interleaving is not exhaustively explored. That is what needs a
|
|
131
|
+
person — the test itself is fine to run.
|
|
132
132
|
|
|
133
|
-
##
|
|
133
|
+
## A `candidate` record
|
|
134
|
+
|
|
135
|
+
Same file, same path, and it runs:
|
|
134
136
|
|
|
135
137
|
```python
|
|
136
|
-
@pytest.mark.skip(reason="spec-correctness review pending")
|
|
137
138
|
@given(src=plugin_sources)
|
|
138
139
|
def test_fr_018_ac_3_source_maps_to_one_root(src):
|
|
139
140
|
"""Trace: FR-018-AC-3 — a plugin source maps to exactly one resolved root.
|
|
140
141
|
|
|
141
|
-
spec-correctness: row=FR-018-AC-3 property=invariant extraction=candidate origin=regex-candidate
|
|
142
|
+
spec-correctness: row=FR-018-AC-3 property=invariant extraction=candidate origin=regex-candidate
|
|
142
143
|
"""
|
|
143
144
|
```
|
|
144
145
|
|
|
145
|
-
|
|
146
|
-
|
|
146
|
+
What marks it for review is a `medium` finding in the review artifact (step 6), not a skip
|
|
147
|
+
marker in the tree.
|
|
147
148
|
|
|
148
149
|
## Notes
|
|
149
150
|
|
|
@@ -152,4 +153,4 @@ change.
|
|
|
152
153
|
- Bound every `st.lists` with `max_size`, every `st.text` with `max_size`.
|
|
153
154
|
- Never `@seed(...)` a generated test, and never `@settings(derandomize=True)` — both hide
|
|
154
155
|
the failures the property exists to find.
|
|
155
|
-
- Raise `max_examples` only with a reason recorded
|
|
156
|
+
- Raise `max_examples` only with a reason recorded as a finding.
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
# Rust / proptest templates
|
|
2
2
|
|
|
3
|
-
File: `tests/props_fr_NNN.rs
|
|
4
|
-
|
|
3
|
+
File: `tests/props_fr_NNN.rs`.
|
|
4
|
+
Every emitted test runs — there is no inert form.
|
|
5
5
|
|
|
6
6
|
Paths are **flat**: cargo auto-discovers integration tests only at `tests/*.rs`. A file
|
|
7
7
|
under `tests/props/` compiles as a helper module and never runs. See step 4.
|
|
@@ -115,20 +115,23 @@ fn fr_050_ac_1_concurrent_writes_linearizable() {
|
|
|
115
115
|
|
|
116
116
|
Keep the loom model tiny — two threads, two ops. Interleavings explode.
|
|
117
117
|
|
|
118
|
-
##
|
|
118
|
+
## A `candidate` record
|
|
119
|
+
|
|
120
|
+
Same file, same path, and it **runs** — `extraction: candidate` means the metamorphic label
|
|
121
|
+
was not corroborated structurally, which is something for a reviewer to read, not a reason
|
|
122
|
+
to disable a test:
|
|
119
123
|
|
|
120
124
|
```rust
|
|
121
125
|
proptest! {
|
|
122
126
|
/// Trace: FR-018-AC-3 — a plugin source maps to exactly one resolved root.
|
|
123
|
-
/// spec-correctness: row=FR-018-AC-3 property=invariant extraction=candidate origin=regex-candidate
|
|
127
|
+
/// spec-correctness: row=FR-018-AC-3 property=invariant extraction=candidate origin=regex-candidate
|
|
124
128
|
#[test]
|
|
125
|
-
#[ignore = "spec-correctness review pending"]
|
|
126
129
|
fn fr_018_ac_3_source_maps_to_one_root(src in plugin_source()) { … }
|
|
127
130
|
}
|
|
128
131
|
```
|
|
129
132
|
|
|
130
|
-
|
|
131
|
-
|
|
133
|
+
What marks it for review is a `medium` finding in the review artifact (step 6), not an
|
|
134
|
+
`#[ignore]` in the tree.
|
|
132
135
|
|
|
133
136
|
## Notes
|
|
134
137
|
|
|
@@ -140,5 +143,5 @@ change.
|
|
|
140
143
|
- Bound every `prop::collection::vec` — `0..32` unless the FR states a bound.
|
|
141
144
|
- Prefer building a negative domain directly over `prop_filter` when the filter rejects
|
|
142
145
|
most values; proptest gives up after too many rejections.
|
|
143
|
-
- Do not set `ProptestConfig { cases, .. }` above the default without recording why
|
|
144
|
-
|
|
146
|
+
- Do not set `ProptestConfig { cases, .. }` above the default without recording why as a
|
|
147
|
+
finding.
|
|
@@ -38,7 +38,7 @@ Field notes that matter:
|
|
|
38
38
|
|
|
39
39
|
- `row_id` — the only identifier shared with the matrix and with `gap-analysis`. Never
|
|
40
40
|
synthesize one, never alter one. A record with `row_id: null` cannot be tagged, so it
|
|
41
|
-
|
|
41
|
+
yields no test and one finding with reason `no-row-id`.
|
|
42
42
|
- `line` is 1-based and **file**-relative. Span `start`/`end` are **statement**-relative
|
|
43
43
|
byte offsets into `statement`. Do not mix them.
|
|
44
44
|
- Spans are usually `null` (they reach ~4% of the ecosystem corpus). Their absence says
|
|
@@ -63,16 +63,16 @@ and runner.
|
|
|
63
63
|
Rules:
|
|
64
64
|
|
|
65
65
|
- **Manifest present, generator library absent** — install nothing, and **write no test
|
|
66
|
-
files**. A file that imports a missing generator library breaks the suite
|
|
67
|
-
|
|
68
|
-
|
|
66
|
+
files**. A file that imports a missing generator library breaks the suite at collection
|
|
67
|
+
time, before a single test runs. Record one `high` finding per affected criterion naming
|
|
68
|
+
the missing library, and report one line:
|
|
69
69
|
`harness <lib> not installed — add <lib> to <dev-deps>, then re-run to emit these`.
|
|
70
70
|
Adding a dependency is the user's call, not this skill's.
|
|
71
71
|
- **Polyglot repo** — resolve per-FR from the AC table's `Verification` column, which names
|
|
72
72
|
the test file (`Test (org.test.ts)` → TypeScript). If that column is absent, use the tree
|
|
73
73
|
holding the symbol step 2 grounded. Never pick one harness repo-wide by guessing.
|
|
74
|
-
- **No harness at all** — stop after steps 1 and 2.
|
|
75
|
-
in the
|
|
74
|
+
- **No harness at all** — stop after steps 1 and 2. Record the grounded strategies as
|
|
75
|
+
findings in the review artifact. Write no test files.
|
|
76
76
|
|
|
77
77
|
## 4. Detect the existing tag style
|
|
78
78
|
|
|
@@ -36,7 +36,7 @@ extractable" and never "consider rewriting the 59 example criteria as properties
|
|
|
36
36
|
## What is worth flagging (as information)
|
|
37
37
|
|
|
38
38
|
- Records with `row_id: null` — they cannot be tagged, so they cannot be reconciled. Route
|
|
39
|
-
|
|
39
|
+
recorded as a finding with reason `no-row-id`.
|
|
40
40
|
- Documents that produced zero criteria — usually the archetype binds no `ac` grammar
|
|
41
41
|
(US and IT are unbound), which is expected, not a gap.
|
|
42
42
|
- A `candidate` count above zero — rare (0.4% of the corpus). Worth naming because every
|
|
@@ -100,11 +100,11 @@ witness: *"A repeated module root is loaded only once"* quantifies; *"A bareword
|
|
|
100
100
|
|
|
101
101
|
The engine cannot tell these apart without giving up determinism, and it is not supposed
|
|
102
102
|
to — that is what this step is for. Ground the domain: if it has one element, the criterion
|
|
103
|
-
is a witness.
|
|
103
|
+
is a witness. Emit it as a `Unit` test (step 5's *witness* outcome) and record a finding, not as an
|
|
104
104
|
unattended property.
|
|
105
105
|
|
|
106
106
|
This is a **routing** decision, not a complaint about the criterion, and never a reason to
|
|
107
|
-
suggest rewording. Record it as `singleton-domain`
|
|
107
|
+
suggest rewording. Record it as a `singleton-domain` finding.
|
|
108
108
|
|
|
109
109
|
### `label-from-mention`
|
|
110
110
|
|
|
@@ -13,7 +13,7 @@ in the sentence.
|
|
|
13
13
|
|
|
14
14
|
| | `extractable` | `candidate` | `not-extractable` |
|
|
15
15
|
| --- | --- | --- | --- |
|
|
16
|
-
| One of the 8 generatable properties | emit
|
|
16
|
+
| One of the 8 generatable properties | emit, no finding | emit **+ a finding** | grounded → emit + finding; refused → step 5 |
|
|
17
17
|
| `example` / `unclassified` | n/a | n/a | **step 5 only** |
|
|
18
18
|
|
|
19
19
|
**Grounding outranks the table.** The lanes above assume step 2 grounded the criterion as
|
|
@@ -22,14 +22,15 @@ that instead — a record can be `extraction: extractable` and still not be a pr
|
|
|
22
22
|
|
|
23
23
|
| step-2 reason | lane |
|
|
24
24
|
| --- | --- |
|
|
25
|
-
| `singleton-domain` |
|
|
25
|
+
| `singleton-domain` | emit as a `Unit` **witness** and record a finding — it is not property coverage |
|
|
26
26
|
| `label-from-mention` | re-derive the strategy from the grounded oracle; lane unchanged |
|
|
27
27
|
| `criterion-describes-its-test` | find the existing test; record *already covered*, emit nothing |
|
|
28
28
|
| `static-or-demonstration` | refuse; no matrix row |
|
|
29
|
-
| any other refusal |
|
|
29
|
+
| any other refusal | no test; the reason becomes a finding, or goes to step 5 |
|
|
30
30
|
|
|
31
31
|
`singleton-domain` is the common one. On a real run it moved 31 of 52 `universal` criteria
|
|
32
|
-
out of the
|
|
32
|
+
out of the property lane and into witnesses. Routing on `extraction` alone would have
|
|
33
|
+
claimed all 52 as property coverage.
|
|
33
34
|
|
|
34
35
|
`candidate` means the criterion carries a metamorphic label the structural pass did not
|
|
35
36
|
corroborate — the label came from a declared idiom alone, or the shape landed but no
|
|
@@ -55,13 +56,14 @@ Full table with generator notes: [`../assets/strategy-table.md`](../assets/strat
|
|
|
55
56
|
| `unclassified` | — | not a property. Step 5 only |
|
|
56
57
|
|
|
57
58
|
`universal` is the default and the largest bucket. `round-trip` needs **two** symbols — if
|
|
58
|
-
grounding found only one side of the pair,
|
|
59
|
-
inverse.
|
|
59
|
+
grounding found only one side of the pair, write no test and record the reason as a finding
|
|
60
|
+
rather than inventing the inverse.
|
|
60
61
|
|
|
61
62
|
## Harness-specific downgrades
|
|
62
63
|
|
|
63
64
|
- **`concurrency` in Python** — hypothesis ships no interleaving scheduler. Emit a
|
|
64
|
-
`RuleBasedStateMachine` sketch over the async model, always
|
|
65
|
+
`RuleBasedStateMachine` sketch over the async model, always with a finding recording the
|
|
66
|
+
downgrade.
|
|
65
67
|
- **`lifecycle` anywhere** — generatable only if the states and transitions are enumerable
|
|
66
68
|
from the FR's `## Behavior`. Otherwise the refusal is `no-state-machine`.
|
|
67
69
|
- **`concurrency` in Rust** — prefer `loom` for exhaustive interleaving of a small model;
|
|
@@ -69,8 +71,8 @@ inverse.
|
|
|
69
71
|
|
|
70
72
|
## Weak assertions are a failure, not a fallback
|
|
71
73
|
|
|
72
|
-
If the strategy collapses to any of these, do not emit it —
|
|
73
|
-
|
|
74
|
+
If the strategy collapses to any of these, do not emit it — record the grounding refusal as
|
|
75
|
+
a finding instead:
|
|
74
76
|
|
|
75
77
|
- `assert result is not None` / `expect(x).toBeDefined()` / `assert!(r.is_ok())` alone
|
|
76
78
|
- a type check with no value check
|
|
@@ -79,5 +81,5 @@ refusal instead:
|
|
|
79
81
|
|
|
80
82
|
## Output of this step
|
|
81
83
|
|
|
82
|
-
Per record: `{row_id, strategy, lane, grounding}` — where
|
|
83
|
-
second-pass}`.
|
|
84
|
+
Per record: `{row_id, strategy, lane, grounding}` — where
|
|
85
|
+
`lane ∈ {emit, emit-with-finding, finding-only, second-pass}`.
|
|
@@ -16,14 +16,33 @@ carriers**, so a single formatting change cannot break reconciliation:
|
|
|
16
16
|
2. The test function or `describe` name containing the id —
|
|
17
17
|
`fr_027_ac_1_…` in Rust/Python, `"FR-027-AC-1 …"` in TypeScript.
|
|
18
18
|
|
|
19
|
+
### Where carrier 1 attaches
|
|
20
|
+
|
|
21
|
+
A carrier is not free-floating text. `quire coverage` binds a tag to the **test symbol
|
|
22
|
+
whose source span encloses it** — the span covering that symbol's annotation block, its
|
|
23
|
+
declaration, and its body. Placement is therefore part of the contract, and it differs by
|
|
24
|
+
language:
|
|
25
|
+
|
|
26
|
+
| Harness | Put `Trace:` | Because |
|
|
27
|
+
|---------|--------------|---------|
|
|
28
|
+
| Rust | in the doc comment above `#[test] fn …` | the span starts at the annotation block |
|
|
29
|
+
| Python | in the test function's docstring | the span covers the body |
|
|
30
|
+
| TypeScript | **immediately above `it(` / `test(`** — never above `describe(` | `describe(…)` groups tests but registers no symbol itself |
|
|
31
|
+
|
|
32
|
+
The TypeScript row is the one that bites. A tag above a `describe` block reads correctly,
|
|
33
|
+
passes review, and matches a grep — and binds to nothing at all. Six generated files
|
|
34
|
+
shipped that way in `@agent-ix/quoin@0.12.x` and every criterion in them scored zero
|
|
35
|
+
(agent-ix/quoin#61). **Do not verify placement with grep**: grep does not care where a
|
|
36
|
+
comment sits, which is exactly why the defect shipped.
|
|
37
|
+
|
|
19
38
|
Plus one provenance line, ignored by the reconciliation grep and read by this skill on
|
|
20
39
|
re-runs:
|
|
21
40
|
|
|
22
41
|
```
|
|
23
|
-
spec-correctness: row=FR-027-AC-1 property=universal extraction=extractable origin=regex
|
|
42
|
+
spec-correctness: row=FR-027-AC-1 property=universal extraction=extractable origin=regex
|
|
24
43
|
```
|
|
25
44
|
|
|
26
|
-
`origin ∈ {regex, regex-candidate, llm-second-pass}`.
|
|
45
|
+
`origin ∈ {regex, regex-candidate, llm-second-pass}`.
|
|
27
46
|
For `origin=llm-second-pass`, add `confidence=<high|medium|low>`.
|
|
28
47
|
|
|
29
48
|
Hard rules:
|
|
@@ -46,11 +65,14 @@ The provenance line is the key. On a re-run:
|
|
|
46
65
|
|
|
47
66
|
## File placement
|
|
48
67
|
|
|
49
|
-
| Harness |
|
|
50
|
-
| --- | --- |
|
|
51
|
-
| Rust | `tests/props_fr_NNN.rs` |
|
|
52
|
-
| TypeScript | `tests/props/fr-NNN.prop.test.ts` |
|
|
53
|
-
| Python | `tests/props/test_fr_NNN.py` |
|
|
68
|
+
| Harness | Path |
|
|
69
|
+
| --- | --- |
|
|
70
|
+
| Rust | `tests/props_fr_NNN.rs` |
|
|
71
|
+
| TypeScript | `tests/props/fr-NNN.prop.test.ts` |
|
|
72
|
+
| Python | `tests/props/test_fr_NNN.py` |
|
|
73
|
+
|
|
74
|
+
One placement, because there is one kind of emitted test. A criterion that cannot be
|
|
75
|
+
grounded gets **no file at all** — it gets a finding in the review artifact (step 6).
|
|
54
76
|
|
|
55
77
|
**Rust paths are flat on purpose.** Cargo auto-discovers integration tests only at
|
|
56
78
|
`tests/*.rs`; a file under `tests/props/` is treated as a helper module and is never
|
|
@@ -61,25 +83,21 @@ prefix keep the grouping without the manifest edit.
|
|
|
61
83
|
For a Rust repo that keeps `#[cfg(test)]` unit tests inline and no `tests/` dir at all,
|
|
62
84
|
still write `tests/props_fr_NNN.rs` — an integration test only sees the public API, which is
|
|
63
85
|
the right boundary for a criterion-derived property anyway. If the symbol under test is
|
|
64
|
-
private, that is a grounding result, not a placement problem:
|
|
65
|
-
|
|
86
|
+
private, that is a grounding result, not a placement problem: write no test and record
|
|
87
|
+
`symbol-not-public` as a finding.
|
|
66
88
|
|
|
67
89
|
One file per FR, one test per `row_id`. Shared generators go in a sibling `arbitraries`
|
|
68
90
|
module rather than being duplicated per test.
|
|
69
91
|
|
|
70
|
-
##
|
|
92
|
+
## Every emitted test runs
|
|
71
93
|
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
| --- | --- |
|
|
76
|
-
| Rust | `#[ignore = "spec-correctness review pending"]` |
|
|
77
|
-
| TypeScript | `it.skip(…)` / `describe.skip(…)` |
|
|
78
|
-
| Python | `@pytest.mark.skip(reason="spec-correctness review pending")` |
|
|
94
|
+
No skip marker, no `#[ignore]`, no `_review/` directory. A disabled test checked into a
|
|
95
|
+
repo is a dead test, and the review it was waiting for already happens in the pull request
|
|
96
|
+
the test arrives in.
|
|
79
97
|
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
98
|
+
Whether a matrix row reads `✅` is decided by a real run and set by `spec-matrix` — not
|
|
99
|
+
pre-empted here with a marker. A criterion that is not ready to be tested is not a disabled
|
|
100
|
+
test; it is a finding (step 6).
|
|
83
101
|
|
|
84
102
|
## Generator hygiene
|
|
85
103
|
|
|
@@ -89,11 +107,11 @@ one-line diff plus a move.
|
|
|
89
107
|
- Seed generators so both branches of a partitioned domain are reachable — check the
|
|
90
108
|
sibling ACs from step 2.
|
|
91
109
|
- Do not set a fixed RNG seed. Do not raise the case count above the harness default
|
|
92
|
-
without a reason recorded
|
|
110
|
+
without a reason recorded as a finding.
|
|
93
111
|
|
|
94
112
|
## After writing
|
|
95
113
|
|
|
96
|
-
Run the suite. A failing
|
|
97
|
-
and not a code bug — fix the grounding or
|
|
98
|
-
weaken an assertion to make a generated test pass,
|
|
99
|
-
generated test.
|
|
114
|
+
Run the suite. A failing generated test is a grounding bug in step 2, not a spec bug
|
|
115
|
+
and not a code bug — fix the grounding, or delete the test and record the reason as a
|
|
116
|
+
finding. Never weaken an assertion to make a generated test pass, never disable it, and
|
|
117
|
+
never edit the spec to match a generated test.
|
|
@@ -26,7 +26,7 @@ file, the sibling ACs, and the located symbol if there is one.
|
|
|
26
26
|
|
|
27
27
|
1. **Reclassify** into one of the 8 generatable families, with a proposed domain,
|
|
28
28
|
precondition and oracle, each carrying at least one `file:line` citation. → a property
|
|
29
|
-
test
|
|
29
|
+
test, plus a finding recording that an LLM read it.
|
|
30
30
|
|
|
31
31
|
Worth looking for: a criterion phrased about one concrete input that actually holds for
|
|
32
32
|
a whole class ("Repeated `--types` accumulate into an ordered list" is `ordering`, not an
|
|
@@ -34,10 +34,11 @@ file, the sibling ACs, and the located symbol if there is one.
|
|
|
34
34
|
|
|
35
35
|
2. **Witness** — the criterion genuinely describes a single case, or a small closed set of
|
|
36
36
|
them ("`version`, `--version`, and `-v` each print the package version" is a 3-case
|
|
37
|
-
witness). → an example-based test, `Type: Unit`,
|
|
38
|
-
property coverage** in the
|
|
37
|
+
witness). → an example-based test, `Type: Unit`, with a finding, and **explicitly not
|
|
38
|
+
counted as property coverage** in the census.
|
|
39
39
|
|
|
40
|
-
3. **Nothing** —
|
|
40
|
+
3. **Nothing** — no test; the reason becomes a `medium` finding. It records what could not
|
|
41
|
+
be settled, and is not a judgement on the criterion (CON-1).
|
|
41
42
|
|
|
42
43
|
## It must refuse to
|
|
43
44
|
|
|
@@ -52,18 +53,19 @@ file, the sibling ACs, and the located symbol if there is one.
|
|
|
52
53
|
- **Write a framework name into any spec artifact** (CON-2).
|
|
53
54
|
- **Alter or synthesize a `row_id`.** Every emitted tag must be a `row_id` present in the
|
|
54
55
|
classification output.
|
|
55
|
-
- **
|
|
56
|
+
- **Present its output as deterministic.** Every test from this pass carries
|
|
57
|
+
`origin=llm-second-pass` and a finding; the reviewer must be able to tell an LLM read it.
|
|
56
58
|
|
|
57
59
|
## Marking
|
|
58
60
|
|
|
59
61
|
Every product of this pass:
|
|
60
62
|
|
|
61
63
|
```
|
|
62
|
-
spec-correctness: row=FR-005-AC-2 property=example extraction=not-extractable origin=llm-second-pass
|
|
64
|
+
spec-correctness: row=FR-005-AC-2 property=example extraction=not-extractable origin=llm-second-pass confidence=medium
|
|
63
65
|
```
|
|
64
66
|
|
|
65
|
-
|
|
66
|
-
|
|
67
|
+
The test is written to the normal path and **runs** like any other. Its finding in the
|
|
68
|
+
review artifact is what marks it as needing a read.
|
|
67
69
|
|
|
68
70
|
`extraction: candidate` records use the identical mechanism with `origin=regex-candidate`
|
|
69
71
|
and no `confidence` key — they came from the deterministic pass, not from this one.
|
|
@@ -74,6 +76,6 @@ and no `confidence` key — they came from the deterministic pass, not from this
|
|
|
74
76
|
- `medium` — the oracle was assembled from `## Behavior` plus the code, and reads as a fair
|
|
75
77
|
restatement.
|
|
76
78
|
- `low` — the oracle is a plausible reading among more than one. Say what the alternative
|
|
77
|
-
reading was, in the
|
|
79
|
+
reading was, in the finding's summary.
|
|
78
80
|
|
|
79
81
|
Anything below `low` is proposal 3: nothing.
|
|
@@ -0,0 +1,121 @@
|
|
|
1
|
+
# Step 6: The Review Artifact
|
|
2
|
+
|
|
3
|
+
**Goal**: the run leaves a record of **what it could not ground, and why**, as a validated
|
|
4
|
+
artifact a reviewer can read without re-deriving anything.
|
|
5
|
+
|
|
6
|
+
The tests it *could* write are already under review — they arrive in a pull request, which
|
|
7
|
+
is what a review gate is. What a PR does not carry is the reasoning behind a criterion the
|
|
8
|
+
run declined to test. That, and only that, is what this artifact is for.
|
|
9
|
+
|
|
10
|
+
## Where it lives
|
|
11
|
+
|
|
12
|
+
`reviews/YY-MM-DD-<slug>.md` at the repo root — the same location and archetype
|
|
13
|
+
`gap-analysis` uses. It is a **`SpecReview`** with `analysis: spec-correctness`.
|
|
14
|
+
|
|
15
|
+
> Earlier versions of this skill wrote `tests/props/QUEUE.md` instead, because
|
|
16
|
+
> `SpecReview.analysis` had no `spec-correctness` value and a file under `reviews/` would
|
|
17
|
+
> have failed `quire validate`. A closed enum with no fitting value is the system saying an
|
|
18
|
+
> artifact type is missing; it was read as an obstacle and routed around, and what it
|
|
19
|
+
> produced was an unvalidated report committed into a test tree. The enum was fixed in
|
|
20
|
+
> agent-ix/spec-artifacts-process#11. **Do not invent an output format** (FR-028-CON-1) —
|
|
21
|
+
> if nothing fits, that is a ticket, not a new filename.
|
|
22
|
+
|
|
23
|
+
## Fetch the template, then author
|
|
24
|
+
|
|
25
|
+
```
|
|
26
|
+
quoin write --types SpecReview
|
|
27
|
+
```
|
|
28
|
+
|
|
29
|
+
If `quoin write` is unavailable in the working tree, read the skeleton directly from
|
|
30
|
+
`~/.ix/filament/modules/spec-artifacts-process/skeletons/SpecReview.md`.
|
|
31
|
+
|
|
32
|
+
## Frontmatter
|
|
33
|
+
|
|
34
|
+
```yaml
|
|
35
|
+
---
|
|
36
|
+
id: SR-002 # ^[A-Z]{2,4}-[0-9]+$ — bump past any existing SR-
|
|
37
|
+
title: "spec-correctness — <repo> — <N> criteria"
|
|
38
|
+
type: SpecReview
|
|
39
|
+
analysis: spec-correctness # the dedicated analysis value
|
|
40
|
+
scope: "spec/**/*.md, tests/props/"
|
|
41
|
+
review_set: all
|
|
42
|
+
---
|
|
43
|
+
```
|
|
44
|
+
|
|
45
|
+
## Body
|
|
46
|
+
|
|
47
|
+
`## Summary` and `## Findings` are **required and validated**; the rest are extra sections.
|
|
48
|
+
|
|
49
|
+
```markdown
|
|
50
|
+
## Summary
|
|
51
|
+
|
|
52
|
+
<1–2 sentences: what corpus was classified, how many criteria, and how many are now
|
|
53
|
+
covered by a generated test.>
|
|
54
|
+
|
|
55
|
+
## Findings
|
|
56
|
+
|
|
57
|
+
| ID | Severity | Summary | Refs |
|
|
58
|
+
| ------- | -------- | ------------------------------------------------------------- | ------------ |
|
|
59
|
+
| FND-001 | medium | Candidate label not corroborated structurally — verify the shape | FR-018-AC-3 |
|
|
60
|
+
| FND-002 | medium | Oracle is adjectival ("clear error"); nothing to assert | FR-022-AC-1 |
|
|
61
|
+
| FND-003 | low | Domain is a single value; a Unit witness, not property coverage | FR-009-AC-2 |
|
|
62
|
+
| FND-004 | high | No generator library in the manifest; no test written | FR-031-AC-1 |
|
|
63
|
+
|
|
64
|
+
## Census
|
|
65
|
+
|
|
66
|
+
emitted 37 extractable, grounded
|
|
67
|
+
reviewed 6 emitted and flagged (`extraction: candidate`)
|
|
68
|
+
not settled 14 symbol-not-found 6 · oracle-is-adjectival 5 · unimplemented 3
|
|
69
|
+
already covered 9 hand-written tests already carry the row_id
|
|
70
|
+
witnesses 5 Unit tests, not property coverage
|
|
71
|
+
```
|
|
72
|
+
|
|
73
|
+
### Findings table contract (validated)
|
|
74
|
+
|
|
75
|
+
- Headers EXACTLY: `ID | Severity | Summary | Refs`.
|
|
76
|
+
- `ID` matches `^FND-\d+$`; **≥1 row**.
|
|
77
|
+
- `Severity` ∈ `low | medium | high`.
|
|
78
|
+
- A run that grounded everything still records one row:
|
|
79
|
+
`FND-001 | low | Every binding criterion was grounded | -`.
|
|
80
|
+
|
|
81
|
+
## What each finding is
|
|
82
|
+
|
|
83
|
+
| Reason | Severity | Note |
|
|
84
|
+
| --- | --- | --- |
|
|
85
|
+
| `extraction: candidate` | medium | A test **was** emitted; the metamorphic label was not corroborated structurally, so the reviewer checks the shape |
|
|
86
|
+
| Second-pass reclassify | medium | An LLM read, not a deterministic one |
|
|
87
|
+
| Second-pass witness | low | An example test, not property coverage |
|
|
88
|
+
| `singleton-domain` witness | low | Grounding found a one-element domain — a `Unit` test |
|
|
89
|
+
| Harness-downgraded | low | e.g. Python `concurrency` |
|
|
90
|
+
| Not grounded | medium | No test written; the reason **is** the finding |
|
|
91
|
+
| Generator library missing | high | No test written; name the library and the remedy |
|
|
92
|
+
|
|
93
|
+
## Severity is about the reader's next action, not about quality
|
|
94
|
+
|
|
95
|
+
`high` means a person must do something before this criterion can be covered at all —
|
|
96
|
+
install a library, implement a symbol. `medium` means read the generated test and confirm
|
|
97
|
+
it. `low` means know that a row is a witness rather than a property. None of it is a
|
|
98
|
+
verdict on the spec (FR-052-CON-1), and none of it recommends rewording a criterion.
|
|
99
|
+
|
|
100
|
+
## Emitted tests are not disabled
|
|
101
|
+
|
|
102
|
+
Every test this skill writes **runs**. No `describe.skip`, no `#[ignore]`, no
|
|
103
|
+
`@pytest.mark.skip`, no `_review/` directory. A disabled test in a repo is a dead test, and
|
|
104
|
+
whether a matrix row reads `✅` is the `spec-matrix` skill's status column to set from a
|
|
105
|
+
real run — not this skill's to pre-empt with a marker.
|
|
106
|
+
|
|
107
|
+
## Validate
|
|
108
|
+
|
|
109
|
+
```
|
|
110
|
+
quire validate --scope <project_root> "reviews/**/*.md"
|
|
111
|
+
```
|
|
112
|
+
|
|
113
|
+
Fix any validation error — frontmatter pattern, the `analysis` enum, findings
|
|
114
|
+
headers/ids/severity — before reporting completion. Then tell the user the artifact path.
|
|
115
|
+
|
|
116
|
+
## Notes
|
|
117
|
+
|
|
118
|
+
- File name: `reviews/<YYYY-MM-DD>-<short-slug>.md`, today's real date.
|
|
119
|
+
- One run → one `SpecReview`, matching the one-doc-per-analysis model.
|
|
120
|
+
- A re-run reads the previous artifact's findings so it does not re-propose a criterion a
|
|
121
|
+
reviewer already rejected; a criterion whose `statement` has changed is proposed again.
|