specproof 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/CHANGELOG.md +10 -0
- data/LICENSE.txt +21 -0
- data/README.md +283 -0
- data/docs/agent-skills.md +60 -0
- data/docs/benchmark.md +24 -0
- data/docs/constrained-partition-pairs-results.md +44 -0
- data/docs/finite-strings-results.md +75 -0
- data/docs/implementation-status.md +65 -0
- data/docs/input-selection-results.md +123 -0
- data/docs/lint-cleanup-plan.md +12 -0
- data/docs/method-targets-results.md +68 -0
- data/docs/minitest-compatibility.md +33 -0
- data/docs/optional-arguments-plan.md +25 -0
- data/docs/optional-arguments-results.md +45 -0
- data/docs/pairwise-partitions-results.md +53 -0
- data/docs/pairwise-results.md +77 -0
- data/docs/pairwise-suggestions-results.md +104 -0
- data/docs/rbs-implementation-status.md +115 -0
- data/docs/rbs-string-literals-results.md +58 -0
- data/docs/real-project-pilot.md +84 -0
- data/docs/report-format.md +244 -0
- data/examples/age_policy/README.md +39 -0
- data/examples/age_policy/faulty/lib/age_policy.rb +9 -0
- data/examples/age_policy/faulty/test/age_policy_examples_test.rb +17 -0
- data/examples/age_policy/faulty/test/age_policy_test.rb +21 -0
- data/examples/age_policy/lib/age_policy.rb +8 -0
- data/examples/age_policy/run.rb +23 -0
- data/examples/age_policy/specification.rb +8 -0
- data/examples/age_policy/test/age_policy_test.rb +18 -0
- data/examples/constrained_partition_pairs/README.md +40 -0
- data/examples/constrained_partition_pairs/lib/account_eligibility.rb +8 -0
- data/examples/constrained_partition_pairs/requirements.md +11 -0
- data/examples/constrained_partition_pairs/run.rb +22 -0
- data/examples/constrained_partition_pairs/specification.rb +17 -0
- data/examples/constrained_partition_pairs/test_account_eligibility.rb +34 -0
- data/examples/input_selection/README.md +22 -0
- data/examples/input_selection/delivery_policy.rb +11 -0
- data/examples/input_selection/requirements.md +9 -0
- data/examples/input_selection/specification.rb +8 -0
- data/examples/input_selection/test_delivery_policy.rb +38 -0
- data/examples/method_targets/README.md +30 -0
- data/examples/method_targets/lib/delivery_defaults.rb +8 -0
- data/examples/method_targets/lib/delivery_options.rb +12 -0
- data/examples/method_targets/requirements.md +20 -0
- data/examples/method_targets/specification_objects.rb +17 -0
- data/examples/method_targets/specification_strings.rb +15 -0
- data/examples/pairwise/README.md +46 -0
- data/examples/pairwise/constrained_requirements.md +6 -0
- data/examples/pairwise/constrained_specification.rb +11 -0
- data/examples/pairwise/corrected/lib/delivery_options.rb +11 -0
- data/examples/pairwise/corrected/test/delivery_options_test.rb +28 -0
- data/examples/pairwise/faulty/lib/delivery_options.rb +11 -0
- data/examples/pairwise/faulty/test/gap_delivery_options_test.rb +10 -0
- data/examples/pairwise/faulty/test/initial_delivery_options_test.rb +23 -0
- data/examples/pairwise/requirements.md +11 -0
- data/examples/pairwise/run.rb +36 -0
- data/examples/pairwise/specification.rb +9 -0
- data/examples/pairwise/three_boolean/lib/boolean_batch.rb +8 -0
- data/examples/pairwise/three_boolean/test/boolean_batch_test.rb +28 -0
- data/examples/pairwise/three_boolean_specification.rb +10 -0
- data/examples/pairwise_partitions/README.md +34 -0
- data/examples/pairwise_partitions/lib/age_policy.rb +8 -0
- data/examples/pairwise_partitions/requirements.md +8 -0
- data/examples/pairwise_partitions/run.rb +22 -0
- data/examples/pairwise_partitions/specification.rb +13 -0
- data/examples/pairwise_partitions/test_age_policy.rb +29 -0
- data/examples/pairwise_suggestions/README.md +48 -0
- data/examples/pairwise_suggestions/run.rb +55 -0
- data/examples/pairwise_suggestions/three_boolean/lib/suggested_boolean_batch.rb +8 -0
- data/examples/pairwise_suggestions/three_boolean/requirements.md +7 -0
- data/examples/pairwise_suggestions/three_boolean/specification.rb +11 -0
- data/examples/pairwise_suggestions/three_boolean/test/baseline_test.rb +9 -0
- data/examples/pairwise_suggestions/three_boolean/test/complete_test.rb +22 -0
- data/examples/partition_gap.rb +8 -0
- data/examples/rbs_registration/README.md +58 -0
- data/examples/rbs_registration/implementations/age_gt/registration.rb +8 -0
- data/examples/rbs_registration/implementations/age_gte/registration.rb +8 -0
- data/examples/rbs_registration/requirements.md +9 -0
- data/examples/rbs_registration/sig/registration.rbs +3 -0
- data/examples/rbs_registration/specification.rb +10 -0
- data/examples/rbs_registration/test_registration.rb +43 -0
- data/examples/rbs_string_literals/README.md +34 -0
- data/examples/rbs_string_literals/lib/delivery_options.rb +9 -0
- data/examples/rbs_string_literals/requirements.md +11 -0
- data/examples/rbs_string_literals/sig/delivery_options.rbs +3 -0
- data/examples/rbs_string_literals/specification.rb +7 -0
- data/examples/rbs_string_literals/test_delivery_options.rb +46 -0
- data/examples/shipping.rb +12 -0
- data/exe/specproof +7 -0
- data/lib/specproof/checker.rb +144 -0
- data/lib/specproof/cli.rb +292 -0
- data/lib/specproof/collector.rb +304 -0
- data/lib/specproof/combinations.rb +226 -0
- data/lib/specproof/composer.rb +97 -0
- data/lib/specproof/coverage.rb +229 -0
- data/lib/specproof/coverage_report.rb +334 -0
- data/lib/specproof/domains.rb +124 -0
- data/lib/specproof/loader.rb +285 -0
- data/lib/specproof/minitest.rb +443 -0
- data/lib/specproof/model.rb +198 -0
- data/lib/specproof/partition_space.rb +268 -0
- data/lib/specproof/planner.rb +125 -0
- data/lib/specproof/report.rb +248 -0
- data/lib/specproof/skills.rb +188 -0
- data/lib/specproof/sources/rbs.rb +137 -0
- data/lib/specproof/sources/rbs_projector.rb +268 -0
- data/lib/specproof/suggestion_report.rb +159 -0
- data/lib/specproof/suggestions.rb +352 -0
- data/lib/specproof/target_name.rb +100 -0
- data/lib/specproof/type_constraints.rb +76 -0
- data/lib/specproof/version.rb +7 -0
- data/lib/specproof.rb +100 -0
- data/sig/specproof.rbs +222 -0
- data/skills/specproof-specify/SKILL.md +282 -0
- data/skills/specproof-specify/references/usage.md +265 -0
- data/skills/specproof-test-gaps/SKILL.md +214 -0
- data/skills/specproof-test-gaps/references/usage.md +231 -0
- metadata +165 -0
|
@@ -0,0 +1,115 @@
|
|
|
1
|
+
# RBS scalar integration results
|
|
2
|
+
|
|
3
|
+
This page records the RBS scalar integration as delivered on 2026-10-02. Its
|
|
4
|
+
implementation slices, verification counts, pilots, and limitations below are
|
|
5
|
+
historical evidence for that change. The method-target extension delivered on
|
|
6
|
+
2026-10-06 supersedes the target-form limitations described in the historical
|
|
7
|
+
sections; those earlier artifacts are retained as originally recorded.
|
|
8
|
+
|
|
9
|
+
The approved [plan](superpowers/plans/2026-10-02-rbs-scalar-integration.md), commit `49a233a`, is implemented on `implement/rbs-scalar-integration` in the isolated `tmp/worktrees/rbs-scalar-integration` worktree. The original checkout remains on `implement/first-usable-release`. No push, publication, or merge is included.
|
|
10
|
+
|
|
11
|
+
## Delivered behavior
|
|
12
|
+
|
|
13
|
+
An optional RBS 4.2.0 reader imports explicitly selected, directly declared public class-instance input signatures. Scalar type facts compose with existing DSL domains; checking, planning, and collection consume the source-neutral IR. Finite inputs work without duplicate declarations; unrestricted integers require behavioral refinements. Unsupported selected types fail atomically with named limitations.
|
|
14
|
+
|
|
15
|
+
Refinements may narrow imported types, but incompatible admitted values remain visible. Checking reports a concrete witness and both declaration sources; planning rejects conflicts. Integer compatibility checks scan declared members rather than enumerating ranges, including integers beyond machine width. Equivalent compiled contracts retain existing obligation IDs, criterion, and behavioral digest.
|
|
16
|
+
|
|
17
|
+
Public loading, repeatable CLI `--rbs`/`--target` options, and Minitest installation accept the same source selection. Reports distinguish canonical supplied declarations from all resulting runtime targets. Preparation failures preserve original tests and mark analysis incomplete. Collection observes effective bound values; argument omission and output behavior are not measured.
|
|
18
|
+
|
|
19
|
+
For a target selected from RBS, extra DSL input names raise
|
|
20
|
+
`rbs_input_mismatch` before planning, with both declaration locations. This
|
|
21
|
+
adapter-level check preserves generic composition of partial type facts.
|
|
22
|
+
Unselected DSL targets and correctly named partial refinements remain valid.
|
|
23
|
+
|
|
24
|
+
The core gem retains **zero runtime dependencies**. The default Gemfile and lockfile are unchanged. RBS lives in the opt-in `Gemfile.rbs`; core checks/plans and skill export work without it.
|
|
25
|
+
|
|
26
|
+
## Implementation slices
|
|
27
|
+
|
|
28
|
+
| Task | Result |
|
|
29
|
+
| --- | --- |
|
|
30
|
+
| 1 | `765db3e`: immutable scalar constraints/facts, provenance, nonenumerating interval checks |
|
|
31
|
+
| 2 | `c4dd0e3`: composition, duplicate rejection, conflict checking, shared planning |
|
|
32
|
+
| 3 | `ae121b2`: real-parser reader and private projector with explicit supported shapes |
|
|
33
|
+
| 4 | `b465d5c`: public API/CLI, RBS-only and mixed loading, API signatures |
|
|
34
|
+
| 5 | `53b379d`: Minitest source integration, declaration scope, preserved failure lifecycle |
|
|
35
|
+
| Review fix | `c0d32ad`: reject absent selected-RBS input bindings before offline planning |
|
|
36
|
+
| 6 | `fbe97f6`: Registration example, updated packaged skills, package isolation, six fresh Luna evaluations |
|
|
37
|
+
| 7 | Separate CI lane, three-Ruby matrix, unchanged-RBS project pilot, manifests and final audit |
|
|
38
|
+
|
|
39
|
+
Code slices received independent Luna specification and quality reviews. Scalar constraints live with their operations in `type_constraints.rb`; input facts and generic provenance remain in `model.rb`. A private projector separates AST conversion from reader orchestration. These are file-placement adjustments. No generic adapter registry, nominal domain system, solver, generated DSL, or default evaluator was introduced.
|
|
40
|
+
|
|
41
|
+
Changed core files are `lib/specproof.rb` and
|
|
42
|
+
`lib/specproof/{model,type_constraints,composer,checker,planner,report,loader}.rb`.
|
|
43
|
+
Integration files are `lib/specproof/sources/{rbs,rbs_projector}.rb`,
|
|
44
|
+
`lib/specproof/{cli,minitest}.rb`, `sig/specproof.rbs`, `Gemfile.rbs`, its lockfile,
|
|
45
|
+
and `.github/workflows/main.yml`. Regression coverage is in the scalar/composer
|
|
46
|
+
tests, `test/rbs/`, and `test/packaging_test.rb`. Packaged workflows and evidence
|
|
47
|
+
live in `skills/`, `examples/rbs_registration/`, and the linked documentation.
|
|
48
|
+
|
|
49
|
+
## Verification
|
|
50
|
+
|
|
51
|
+
| Check | Actual result |
|
|
52
|
+
| --- | --- |
|
|
53
|
+
| Default Ruby 4.0.1 `bundle exec rake` | 156 tests, 786 assertions; zero failures/errors/skips; 95 files linted, no offenses |
|
|
54
|
+
| `ruby bin/check` | Syntax, skill/package checks and real gem build passed; focused 2 tests, 76 assertions |
|
|
55
|
+
| Focused runtime regression | 47 tests, 376 assertions; zero failures/errors/skips |
|
|
56
|
+
| CRuby 3.3.6 / 3.4.7 / 4.0.1 | Each core 156 tests/786 assertions; optional RBS 39 tests/396 assertions; zero failures/errors/skips |
|
|
57
|
+
| API declarations | Optional-bundle `rbs -I sig validate` passed |
|
|
58
|
+
| Package and skills | Real installed gem outside checkout; parser-free core CLI, idempotent export and conflict preservation passed |
|
|
59
|
+
| Guided Luna tasks | Six observed outcomes matched supplied requirements; exact exported skills and 17 installed library files audited |
|
|
60
|
+
|
|
61
|
+
[Compatibility and pilot evidence](pilots/rbs-v0.1/README.md) retains exact commands, raw matrix logs, isolated environments, package versions/paths, and published package hashes. [Agent evaluation evidence](evaluations/rbs-agent-skills-v0.1.md) retains rules, reports, actions, edited files, and archive manifests. Newly added prose-matching tests were removed during review; existing structural skill checks, package checks, and actual behavioral evaluations provide verification.
|
|
62
|
+
|
|
63
|
+
## Distinct demonstrations
|
|
64
|
+
|
|
65
|
+
**Shipped example.** [Registration](../examples/rbs_registration/README.md) imports integer age and nullable boolean consent, declaring only age bounds/partitions. Its independent rule requires age at least 18 and true consent. Baseline tests pass with 8/9 marginal obligations; the added boundary assertion fails against `age > 18` while 9/9 inputs are hit; the corrected implementation passes with a fresh 9/9 report. Checkpoints retain identical IDs/digest and distinct UUIDs. Nine marginal values do not mean nine combinations or output conformance proof.
|
|
66
|
+
|
|
67
|
+
**Guided agent tasks.** Six fresh Luna actors used installed-gem exported skills for finite RBS-only authoring, partial refinement, conflicts, unsupported unions, stale evidence, and a justified boundary fix. The boundary actor's RED assertion stopped its remaining examples, giving 5/9 before a final passing 9/9 run. The stale actor rejected an old passing 9/9 report despite matching input identity, then reported the actual fresh age-18 failure and 5/9 evidence. These differ from the shipped example's separate-test RED run. Explicit selectors and a command helper were supplied; this is not a client-discovery or general reliability claim.
|
|
68
|
+
|
|
69
|
+
**Existing RBS adoption.** BranchProof commit `85611af97461d1714bffffff42a4719952b9cbc4` supplies the unchanged `Branchproof::ComparisonReport#exit_code` signature. RBS-only planning yields two boolean obligations; the unchanged focused test passes 2 tests/8 assertions and exercises both. All 59 snapshot files match original source hashes. No test or gap was manufactured, and no full BranchProof suite result is claimed. Rejected candidates from the seven-location survey are documented separately.
|
|
70
|
+
|
|
71
|
+
## Historical remaining limits at RBS integration
|
|
72
|
+
|
|
73
|
+
Only verified RBS 4.2.0 is supported. Inputs are booleans, nil, supported primitive literals (including finite string literals), supported unions/optionals, and resolved built-in Integer with explicit refinement where needed. Unrestricted `String`, nominal objects, aliases, generics, collections, overloads, inherited/singleton/generated methods, rest/block/unnamed forms, and zero-input targets remain excluded. A selected union containing an unsupported member fails atomically. RBS return types are ignored and provide no expected-result oracle. Full RBS arity/order/subtyping conformance, omission coverage, nullable interval domains, automatic discovery, Sorbet, state analysis, and SMT reasoning remain separate work.
|
|
74
|
+
|
|
75
|
+
## Current method target forms
|
|
76
|
+
|
|
77
|
+
The method-target extension supports canonical string and Ruby method-object
|
|
78
|
+
targets for DSL declarations, including directly defined public instance
|
|
79
|
+
methods on named classes and singleton methods on named classes or modules.
|
|
80
|
+
Equivalent string and object forms normalize to the same target identity,
|
|
81
|
+
digest, and obligation IDs. String declarations stay offline and do not
|
|
82
|
+
autoload owners. Method-object declarations require loaded code and use Ruby
|
|
83
|
+
reflection without invoking the target. RBS selection continues to use
|
|
84
|
+
explicit canonical string selectors and now supports `def self.method` on
|
|
85
|
+
named classes and modules, subject to the existing RBS shape limitations.
|
|
86
|
+
|
|
87
|
+
Method objects are validated at declaration time against the current direct
|
|
88
|
+
definition. Aliases, inherited singleton methods, anonymous owners, and bound
|
|
89
|
+
methods on arbitrary instances remain unsupported. Collection resolves
|
|
90
|
+
canonical names afresh at start and retains the runtime method identity,
|
|
91
|
+
constant binding, redefinition, and duplicate-binding guards. Reports continue
|
|
92
|
+
to use canonical string targets. Navigation from these names remains dependent
|
|
93
|
+
on language-server support.
|
|
94
|
+
|
|
95
|
+
These target-form additions do not change the historical RBS pilot outputs or
|
|
96
|
+
claims. The RBS return type still supplies no expected-result oracle; behavior
|
|
97
|
+
assertions must come from independent requirements.
|
|
98
|
+
|
|
99
|
+
### Finite RBS strings
|
|
100
|
+
|
|
101
|
+
The finite-string extension accepts RBS string literals, their supported finite
|
|
102
|
+
unions, and optional forms. These reuse the exact finite-string input domain:
|
|
103
|
+
strings remain distinct from same-spelling symbols, and no text normalization
|
|
104
|
+
is performed. An unrestricted `String` input is still outside the adapter's
|
|
105
|
+
finite scenario model. Return types remain ignored, so expected results must
|
|
106
|
+
come from an independent requirement. See the [compact DeliveryOptions example](../examples/rbs_string_literals/README.md).
|
|
107
|
+
|
|
108
|
+
Matching digests establish equivalent normalized behavioral input contracts, not current implementation text or correctness. Obtain fresh runtime evidence after any source change. Historical evaluation and pilot artifacts remain unchanged; their original hashes are audited separately. Final independent specification/quality review and evidence audit are recorded in the concluding commits.
|
|
109
|
+
|
|
110
|
+
The [independent final audit](pilots/rbs-v0.1/independent-final-audit.json)
|
|
111
|
+
verified archive/member hashes, all 17 installed library files, four skill
|
|
112
|
+
assets, fresh report identity/scope, three-Ruby logs, published package hashes,
|
|
113
|
+
and 59 unchanged pilot source files. All [125 historical artifact hashes](pilots/rbs-v0.1/historical-original-sha-map.json)
|
|
114
|
+
remain unchanged. Final specification and quality reviews approved the
|
|
115
|
+
input-binding correction and remaining scope; no known blocker remains.
|
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
# RBS string-literal inputs
|
|
2
|
+
|
|
3
|
+
RBS 4.2.0 string literals now supply exact finite input domains to the existing
|
|
4
|
+
checker, planner, Minitest collector and suggestion command. A combinations-only
|
|
5
|
+
DSL declaration can use those imported values without repeating them. Optional
|
|
6
|
+
literal types, supported mixed unions, String/Symbol distinctions, source
|
|
7
|
+
provenance and DSL subset refinements retain the existing domain semantics.
|
|
8
|
+
|
|
9
|
+
The implementation reuses `Domains.supported_string?` and the existing immutable
|
|
10
|
+
type constraints. It adds no dependency, criterion or report schema. Reading
|
|
11
|
+
RBS files explicitly as UTF-8 matches RBS's own environment loader and prevents
|
|
12
|
+
locale-dependent interpretation of Unicode literals. Unrestricted `String`,
|
|
13
|
+
aliases, collections and other unsupported selected union members remain
|
|
14
|
+
rejected. Invalid UTF-8 files return `rbs_parse_error` with their path. Return
|
|
15
|
+
types remain outside analysis.
|
|
16
|
+
|
|
17
|
+
## Demonstrated workflow
|
|
18
|
+
|
|
19
|
+
The [DeliveryOptions example](../examples/rbs_string_literals/README.md) imports
|
|
20
|
+
`country: ("ro" | "de")` and `expedited: bool` from its signature. Three passing
|
|
21
|
+
calls exercise 3/4 pairs. The suggestion identifies `country="de",
|
|
22
|
+
expedited=true`; a separate assertion justified by the written delivery rule
|
|
23
|
+
produces a fresh 4/4 report. This demonstrates a coverage improvement without
|
|
24
|
+
claiming that full pair coverage proves behavior.
|
|
25
|
+
|
|
26
|
+
## Agent guidance
|
|
27
|
+
|
|
28
|
+
A Luna baseline actor read the previous main-branch skills at `53b621d` and
|
|
29
|
+
correctly reported that the literal-union workflow was unsupported. This is
|
|
30
|
+
a coordinator-retained summary, not a benchmark or model attestation.
|
|
31
|
+
|
|
32
|
+
A fresh Luna actor used the built gem's exported skills and the example's
|
|
33
|
+
independent requirement to run check, plan, coverage and suggestions. Its
|
|
34
|
+
baseline had one missing pair; the completed run had none. Separate unrestricted
|
|
35
|
+
`String` and literal-plus-`Object` probes failed atomically. The coordinator
|
|
36
|
+
replayed the workflow against the final installed library and checked its
|
|
37
|
+
library and skill hashes against source. Evidence is retained under
|
|
38
|
+
`evaluations/artifacts/rbs-string-literals/`.
|
|
39
|
+
|
|
40
|
+
## Verification
|
|
41
|
+
|
|
42
|
+
| Check | Result |
|
|
43
|
+
| --- | --- |
|
|
44
|
+
| Core suite on Ruby 3.3.6, 3.4.7 and 4.0.1 | 250 tests / 1,297 assertions per runtime |
|
|
45
|
+
| Optional RBS suite on those runtimes | 53 tests / 490 assertions per runtime |
|
|
46
|
+
| RBS signature validation on those runtimes | Passed |
|
|
47
|
+
| Default locked development bundle | Core suite passed; 130 files linted without offenses |
|
|
48
|
+
| `bin/check` | Syntax checks, 2 packaging tests / 76 assertions, and build passed |
|
|
49
|
+
| Final installed-gem example | Passing baseline 3/4; suggested missing pair; passing completion 4/4 |
|
|
50
|
+
| Unsupported-type probes | `String` and literal-plus-`Object` rejected with exit 2 |
|
|
51
|
+
|
|
52
|
+
[Matrix commands and environments](evaluations/artifacts/rbs-string-literals/verification/commands.json),
|
|
53
|
+
[installed workflow evidence](evaluations/artifacts/rbs-string-literals/agent/command-evidence.json),
|
|
54
|
+
and [installed library/skill hashes](evaluations/artifacts/rbs-string-literals/installed-sha256.json)
|
|
55
|
+
are retained. Historical Ruby snapshots use `.rb.txt` so they do not join the
|
|
56
|
+
active tests or lint scope. Existing method-redefinition fixture warnings and
|
|
57
|
+
the gemspec's missing-homepage warning remain; no dependency or lockfile change
|
|
58
|
+
is part of this feature.
|
|
@@ -0,0 +1,84 @@
|
|
|
1
|
+
# Real-project pilot: configuration lookup and adoption limits
|
|
2
|
+
|
|
3
|
+
The first pilot demonstrated a useful test gap on an existing method, and exposed two concrete adoption limits. It did **not** produce three successful coverage integrations: one target was supported, and two representative targets were rejected. Luna agents performed discovery, used the exported skills, and independently reviewed the evidence. Original repositories were left unchanged.
|
|
4
|
+
|
|
5
|
+
This records the implementation cut at commit `405883e`. The later [optional-argument continuation](optional-arguments-results.md) repeats the PStoreLogger candidate with a new collector and exported skills; the original rejected probe remains historical evidence.
|
|
6
|
+
|
|
7
|
+
## Installation
|
|
8
|
+
|
|
9
|
+
The development bundle now selects the correctly labeled published Minitest 5.26.2 release. The adapter supports `>= 5.26.2` and `< 5.28.0`, while continuing to reject package/runtime minor-version mismatches. See [package checksums and compatibility evidence](minitest-compatibility.md).
|
|
10
|
+
|
|
11
|
+
The pilot used standard Bundler `vendor/bundle` installation, with the verified official package in `vendor/cache`. Sandbox restrictions prevented writing a global executable wrapper, and Bundler could not reach its index host; a normal `bundle install --local` from that package cache succeeded. No dependency source, runtime version constant, or global gem installation was changed. Core development checks use the existing published 5.26.2 installation; Ruby 3.3/3.4 verification used local bundles with ambient Ruby 4 gem paths cleared.
|
|
12
|
+
|
|
13
|
+
Bundler loaded SpecProof from its cached path dependency. Every cached library file matched the pilot implementation by SHA-256; those hashes are in the pilot manifest. The retained archive excludes that generated cache and recreates it from `SPECPROOF_ROOT` during installation.
|
|
14
|
+
|
|
15
|
+
## Three candidate methods
|
|
16
|
+
|
|
17
|
+
| Existing method | Result | Consequence |
|
|
18
|
+
| --- | --- | --- |
|
|
19
|
+
| `QualityGate::Config#fetch(key)` | Supported. Eight shipped symbol keys form an explicitly scoped finite input domain. | Measure and close a gap in its existing configuration tests. |
|
|
20
|
+
| `PStoreLogger#save(data, file_id: nil)` | Collector rejects the optional keyword with `unsupported_parameters`. | No coverage claim. The scalar signature-probe declaration is only a candidate subset of the broader PStore data domain. |
|
|
21
|
+
| `Agent::SessionContext::SemanticCategories.lookup(identifier)` | Declaration rejects class-method syntax with `invalid_target`. | No coverage claim. Do not invent an instance-method wrapper to make the pilot pass. |
|
|
22
|
+
|
|
23
|
+
The [rejected-candidate archive](pilots/2026-10-02/rejected/workspace.tar.gz) retains exact source files, source commits/hashes, original test references, the binding probe, command logs, and errors. The PStore probe used Ruby 3.4.7's standard library because `pstore` is no longer a default gem under Ruby 4; no new dependency was installed. The supported pilot used Ruby 4.0.1 throughout.
|
|
24
|
+
|
|
25
|
+
Discovery also examined SimpleResult, the Minitest reporter, BranchProof, Agent Sessions, Agent Session Context, Agent ACL, and Guardrails for Rails. Many methods require optional parameters, class-method targets, strings, or structured objects outside the first release's supported subset. This was a bounded local survey, not proof that no suitable three-method project exists elsewhere.
|
|
26
|
+
|
|
27
|
+
## QualityGate result
|
|
28
|
+
|
|
29
|
+
Source revision: `8af0893de378cfd8076392d39b874f44987d818c`. All **143 copied project files** retain their original hashes. The pilot runs the unchanged `test/quality_gate/config_test.rb` file, not the whole QualityGate suite or its Rails/external-tool gates. The minimal pilot bundle loads this focused library/test slice without installing QualityGate's broader development dependencies.
|
|
30
|
+
|
|
31
|
+
The domain includes all eight shipped `Config.defaults` keys. It excludes arbitrary hash keys and invalid configuration keys; 100% here means exercise of those eight scoped inputs only.
|
|
32
|
+
|
|
33
|
+
| Run | Tests / assertions | Input obligations exercised |
|
|
34
|
+
| --- | --- | --- |
|
|
35
|
+
| Existing configuration tests | 45 / 342 | 7 / 8; `:files` unobserved |
|
|
36
|
+
| Existing tests plus one added test | 46 / 350 | 8 / 8 |
|
|
37
|
+
|
|
38
|
+
The added test compares `fetch` results with the exact shipped defaults. Its oracle is grounded in the existing test that asserts a default-loaded configuration's `to_h` equals `Config.defaults`. It makes eight assertions and includes obligation IDs and the original assertion location in failure messages. No production behavior was changed or defect invented; these assertions pass against the existing implementation.
|
|
39
|
+
|
|
40
|
+
The fresh final report has a distinct run UUID, unchanged domain/digest and obligation IDs, positive hits for all eight values, and no unknown evidence or integrity errors. Read the [agent's detailed result](pilots/2026-10-02/quality_gate/RESULT.md), [specification](pilots/2026-10-02/quality_gate/config_fetch_spec.rb.txt), [added test](pilots/2026-10-02/quality_gate/config_fetch_added_test.rb.txt), [baseline report](pilots/2026-10-02/quality_gate/baseline-coverage.json), and [final report](pilots/2026-10-02/quality_gate/final-coverage.json).
|
|
41
|
+
|
|
42
|
+
## Timing
|
|
43
|
+
|
|
44
|
+
Five paired subprocess runs executed the **same original 45 tests and 342 assertions**, with no added tests, fixed seed 12345, and the adapter enabled only in the instrumented mode. Every instrumented report retained the same plan, exactly the original `:files` gap, completed integrity, and 27 observed selected-method calls. [Raw timings](pilots/2026-10-02/quality_gate/timings.json) and per-run logs/reports are retained.
|
|
45
|
+
|
|
46
|
+
| Mode | Wall times, seconds | Median |
|
|
47
|
+
| --- | --- | --- |
|
|
48
|
+
| Baseline | 0.307118, 0.300959, 0.298459, 0.300013, 0.318318 | 0.300959 |
|
|
49
|
+
| Instrumented | 0.326725, 0.338711, 0.328865, 0.353294, 0.341357 | 0.338711 |
|
|
50
|
+
|
|
51
|
+
The median difference was about **38 ms (12.5%)** for this focused subprocess workload. Timings include Ruby/Bundler startup, requiring the adapter, collection, and report I/O. Baseline always ran first; five pairs do not establish a general overhead rate or isolate collector cost. They provide a reproducible comparison for this particular suite.
|
|
52
|
+
|
|
53
|
+
## Reproduce
|
|
54
|
+
|
|
55
|
+
The [supported workspace archive](pilots/2026-10-02/quality_gate/workspace.tar.gz) includes unchanged library/tests, license, Gemfile/lock, verified package cache, exported skills, added test, specification, runner, reports, and source hashes. It excludes the installed dependency directory. Unpack it into an empty directory, then use Ruby 3.3 or newer:
|
|
56
|
+
|
|
57
|
+
```sh
|
|
58
|
+
export SPECPROOF_ROOT=/absolute/path/to/gems/specproof
|
|
59
|
+
bundle config set --local path vendor/bundle
|
|
60
|
+
bundle install --local
|
|
61
|
+
export MT_NO_PLUGINS=1
|
|
62
|
+
export PILOT_PROJECT_ROOT="$PWD/project"
|
|
63
|
+
export PILOT_SPECIFICATION="$PWD/config_fetch_spec.rb"
|
|
64
|
+
export PILOT_REPORT="$PWD/fresh-coverage.json"
|
|
65
|
+
PILOT_ADAPTER=1 PILOT_ADDED=1 bundle exec ruby pilot_runner.rb
|
|
66
|
+
```
|
|
67
|
+
|
|
68
|
+
From the SpecProof gem checkout, compare unchanged original tests with:
|
|
69
|
+
|
|
70
|
+
```sh
|
|
71
|
+
ruby script/benchmark_project_pilot.rb /absolute/path/to/unpacked/pilot 5
|
|
72
|
+
```
|
|
73
|
+
|
|
74
|
+
[The manifest](pilots/2026-10-02/manifest.json) records archive/file hashes and the exact updated skill bytes. The earlier twelve-scenario evaluations remain historical evidence for their original assets; this pilot is a separate fresh application of the updated instructions.
|
|
75
|
+
|
|
76
|
+
The retained archive was also unpacked into a second empty directory and installed using `bundle install --local`. A fresh run there passed 46 tests and 350 assertions with 8/8 obligations exercised, confirming that reproduction works independently of the original pilot's installed dependency directory.
|
|
77
|
+
|
|
78
|
+
## Next implementation slice
|
|
79
|
+
|
|
80
|
+
Support **optional scalar positional and keyword parameters** next, while preserving the current directly owned instance-method scope. Define observations as effective bound argument values after Ruby applies defaults. Do not evaluate defaults statically, replay calls, or infer that an observed default value proves the argument was omitted.
|
|
81
|
+
|
|
82
|
+
Before implementing, add TracePoint fixtures proving default evaluation timing, omitted versus explicit equal values, exceptions in default expressions, and unsupported non-primitive effective values. Preserve whole-tuple validation, shared IDs, and unknown evidence. Then rerun the PStoreLogger pilot on an explicitly declared primitive-data scenario subset with independent persistence assertions.
|
|
83
|
+
|
|
84
|
+
Class-method targets, finite string domains, and richer object domains remain separate later decisions. This slice addresses an observed adoption barrier without taking on all of them at once.
|
|
@@ -0,0 +1,244 @@
|
|
|
1
|
+
# Report format
|
|
2
|
+
|
|
3
|
+
Offline commands accept `--format json` and write one JSON document to standard output. Diagnostics and any output printed while evaluating the trusted specification file go to standard error. Human output is the default.
|
|
4
|
+
|
|
5
|
+
Every document includes `schema_version` (`1`), `specproof_version`, `kind`, and `errors`. A check or plan configuration, usage, or I/O error keeps its requested `kind` and puts a stable `code`, explanatory `message`, and optional `path` and `source` in `errors`. An invalid command with no supported command kind uses `kind: "error"`. Every source object requires a file `path` and one-based `line`; schema 1 permits additional fields, and consumers must tolerate unknown source fields. The additive `kind` field identifies source provenance (for example, `dsl` or `rbs`) without changing existing fields or obligation identity.
|
|
6
|
+
|
|
7
|
+
DSL targets may be canonical strings or supported Ruby method objects. Instance
|
|
8
|
+
targets use `Owner#method`; class and module singleton targets use
|
|
9
|
+
`Owner.method`. A directly defined public instance method can be selected with
|
|
10
|
+
`Owner.instance_method(:method)`, and a directly defined public singleton
|
|
11
|
+
method with `Owner.method(:method)`. Equivalent forms normalize to the same
|
|
12
|
+
string target, contract digest, and obligation IDs. String targets remain
|
|
13
|
+
unresolved during offline check and plan, so they do not load or autoload the
|
|
14
|
+
owner. Method objects require the owner code to be loaded and are inspected
|
|
15
|
+
through Ruby reflection without calling the target. They must still represent
|
|
16
|
+
the current direct definition when declared. Aliases, inherited singleton
|
|
17
|
+
methods, anonymous owners, and bound methods on arbitrary instances are
|
|
18
|
+
unsupported; an unbound instance method identifies its defining class.
|
|
19
|
+
Collection resolves each canonical name afresh when it starts and applies its
|
|
20
|
+
existing method-identity and constant-binding guards. Reports retain the
|
|
21
|
+
canonical string target. Navigation to source locations depends on editor and
|
|
22
|
+
language-server support; the report makes no navigation guarantee.
|
|
23
|
+
|
|
24
|
+
Check and plan results include `contract_digest`, a SHA-256 digest of normalized declarations, behavioral input domains, partition policies, and sorted explicitly excluded input names. Exclusion reason, source location, and declaration order do not change the digest. With no exclusions, existing digests and criterion/obligation IDs remain unchanged. Source locations are excluded from this digest. For composed DSL/RBS declarations, the digest fingerprints the resulting behavioral input domains: broader type constraints, provenance, RBS text, parser version, and implementation behavior do not change it. Obtain a fresh runtime report after changing a source or implementation; matching digests establish the same analyzed input obligations and excluded names, not that a particular implementation is correct. Check results include `findings`, each with a semantic `code`, target and input names, a concrete `witness`, and source locations. Findings never appear in `errors`.
|
|
25
|
+
|
|
26
|
+
The DSL options `allow_gaps` and `allow_overlaps` default to `true`; `false`
|
|
27
|
+
requires coverage of the whole input domain or prevents partition overlaps,
|
|
28
|
+
respectively. Legacy `exhaustive` and `disjoint` spellings remain accepted with
|
|
29
|
+
inverted booleans. Equivalent spellings normalize to the same policies and
|
|
30
|
+
preserve digests and obligation IDs; no report schema change is required.
|
|
31
|
+
Supplying both spellings of one policy is a `duplicate_partition_policy`
|
|
32
|
+
configuration error, even when their values express the same requirement.
|
|
33
|
+
|
|
34
|
+
Successful check, plan, and runtime coverage documents share a `scope` object
|
|
35
|
+
with `completeness_scope: "analyzed_inputs"` and `input_scope`. Each input
|
|
36
|
+
scope entry identifies one target, its `analyzed_inputs` names, and its
|
|
37
|
+
`excluded_inputs`, each containing a parameter `name`, a human-readable
|
|
38
|
+
`reason`, and declaration `source` (`path`, one-based `line`, and `kind`). An
|
|
39
|
+
excluded name is part of the contract identity; its reason and source are
|
|
40
|
+
reported for review but do not change identity. Default scope is strict: every
|
|
41
|
+
named runtime parameter must be analyzed or explicitly excluded, and at least
|
|
42
|
+
one must be analyzed. Check/plan scope describes the requested declaration;
|
|
43
|
+
it does not attest runtime binding or test execution.
|
|
44
|
+
|
|
45
|
+
Plan results default to `criterion: "input-basics-v1"` and include `obligations`. Each scalar obligation has a stable `id`, target, input, typed value, merged reasons, and source locations. Values have a `kind`: integer values are decimal strings, symbols are names, booleans are JSON booleans, nil has a JSON null value, and finite strings use `{ "kind": "string", "value": "..." }`. String values are exact text snapshots. This is an additive value kind under schema version 1; consumers must handle it explicitly and tolerate it wherever typed primitive values occur, including witnesses and coverage rows. Strings and same-spelling symbols have distinct identities. Finite string values are deduplicated and ordered deterministically after existing scalar types. Existing nonstring IDs and digests remain unchanged. Existing partition membership, gap, and overlap checks apply to the explicitly enumerated string scenario. Scalar candidates are marginal inputs and do not specify expected method results or form a complete test suite.
|
|
46
|
+
|
|
47
|
+
## Pairwise finite criterion
|
|
48
|
+
|
|
49
|
+
Pairwise planning is opt-in. The Ruby API accepts
|
|
50
|
+
`SpecProof.plan(..., criterion: "pairwise-finite-v1")`; the CLI accepts
|
|
51
|
+
`specproof plan PATH --criterion pairwise-finite-v1`; the Minitest adapter
|
|
52
|
+
accepts `criterion: "pairwise-finite-v1"`. The default remains
|
|
53
|
+
`input-basics-v1`. `check` is criterion-independent and rejects `--criterion`.
|
|
54
|
+
A pairwise plan requires a valid `combinations` group for every supplied target;
|
|
55
|
+
it never skips a target that lacks a group.
|
|
56
|
+
|
|
57
|
+
Each target has at most one group over at least two distinct analyzed finite
|
|
58
|
+
inputs. Select either `all_allowed: true` with no block, or a nonempty block of
|
|
59
|
+
explicit `forbid` assignments. `all_allowed: true` declares that every tuple
|
|
60
|
+
is allowed; it is not evidence that inputs have independent effects. A
|
|
61
|
+
`forbid` clause must be justified by a stated requirement. Clauses are
|
|
62
|
+
conjunctive within a tuple and a tuple is forbidden if any clause matches.
|
|
63
|
+
Partial clauses restrict every completion. They are declarative data and do
|
|
64
|
+
not execute predicates. A forbidden tuple can still contain individually
|
|
65
|
+
feasible pairs when a third input differs; pair feasibility is determined by
|
|
66
|
+
whether the pair has an allowed full group completion.
|
|
67
|
+
|
|
68
|
+
Pairwise plan and runtime rows share `id`, `target`, `reasons`, `sources`,
|
|
69
|
+
`status`, and `hits`, and serialize their two assignments in `inputs` as a
|
|
70
|
+
name-to-typed-value object. Pair names are sorted canonically, while value
|
|
71
|
+
types remain significant; strings and same-spelling symbols are distinct.
|
|
72
|
+
Pairs excluded because no allowed full assignment completes them appear in
|
|
73
|
+
`excluded_pairs`, with `target`, typed `inputs`,
|
|
74
|
+
`reason: "no_allowed_completion"`, and relevant constraint sources. Excluded
|
|
75
|
+
pairs are not denominator rows. Do not remove a missing required pair by
|
|
76
|
+
adding a forbidden rule: the rule must come from an independent requirement.
|
|
77
|
+
|
|
78
|
+
Pairwise planning bounds each target to 10,000 full group assignments and
|
|
79
|
+
10,000 possible pair rows, and bounds all planned pair obligations to 10,000.
|
|
80
|
+
Oversized spaces fail with `combination_limit_exceeded`; SpecProof does not
|
|
81
|
+
sample or truncate. A valid declaration whose restrictions leave no allowed
|
|
82
|
+
full assignment produces the semantic finding `unsatisfiable_combinations`
|
|
83
|
+
and no plan. Malformed declarations are configuration errors. Pairwise
|
|
84
|
+
coverage counts only fully valid analyzed tuples; a forbidden runtime tuple
|
|
85
|
+
fulfills no pair obligations and is reported in the existing outside-domain
|
|
86
|
+
evidence with the added `outside_combinations` evidence. Selected inputs must
|
|
87
|
+
all be valid even when the combination group covers only a subset. Excluded
|
|
88
|
+
runtime inputs remain outside the claim, including any behavior involving
|
|
89
|
+
their values.
|
|
90
|
+
|
|
91
|
+
## Pairwise partition criterion
|
|
92
|
+
|
|
93
|
+
`pairwise-partitions-v1` is a separate explicit criterion. It requires a
|
|
94
|
+
`combinations` group for every target. Each grouped integer input must have a
|
|
95
|
+
bounded range declared `allow_gaps: false` and `allow_overlaps: false`, with
|
|
96
|
+
explicit partitions. Other grouped inputs use finite values as singleton
|
|
97
|
+
categories. A group may declare `all_allowed: true`, or give a nonempty block
|
|
98
|
+
of requirement-backed `forbid` assignments. The default scalar criterion and
|
|
99
|
+
`pairwise-finite-v1` keep their existing behavior.
|
|
100
|
+
|
|
101
|
+
For a restricted group, `in_partition(:name)` selects an entire declared
|
|
102
|
+
integer partition. Exact finite values select singleton categories. A clause
|
|
103
|
+
is a partial assignment: it forbids each matching full assignment, and a tuple
|
|
104
|
+
is forbidden when any clause matches. Integer restrictions cannot name raw
|
|
105
|
+
integer values or partial ranges; selectors must refer to known partitions.
|
|
106
|
+
The selector is distinct from a literal symbol and from a bounded
|
|
107
|
+
`PartitionLevel`. Finite-valued inputs continue to require exact values.
|
|
108
|
+
Clauses are requirement data, never executable predicates.
|
|
109
|
+
|
|
110
|
+
Plan and runtime pair rows use the same `inputs` structure as finite pairwise
|
|
111
|
+
rows. Integer categories serialize as
|
|
112
|
+
`{ "kind": "partition", "name": "minor", "lower": "0", "upper": "17" }`;
|
|
113
|
+
the reported bounds are inclusive. Finite values retain their ordinary typed
|
|
114
|
+
form. Pair IDs include the criterion and category identity and bounds. Coverage
|
|
115
|
+
counts a call when its integer value belongs to the planned category in the
|
|
116
|
+
same fully valid tuple. It does not count as exact-value or boundary coverage,
|
|
117
|
+
and it does not claim equivalent behavior within a category.
|
|
118
|
+
|
|
119
|
+
In a constrained plan, `excluded_pairs` contains category pairs with no
|
|
120
|
+
allowed full completion and `reason: "no_allowed_completion"`. Check all group
|
|
121
|
+
inputs when interpreting feasibility: a restriction on a third input can
|
|
122
|
+
make a pair impossible even when no single clause names both pair members.
|
|
123
|
+
Excluded pairs are outside the required denominator. At runtime, a call that
|
|
124
|
+
matches a forbidden assignment contributes no pair coverage, including pairs
|
|
125
|
+
that would otherwise be valid in some other full assignment. The normalized
|
|
126
|
+
constraint value used for the contract digest encodes a typed restriction
|
|
127
|
+
selector as
|
|
128
|
+
`{ "kind": "partition_reference", "name": "minor" }`; this is separate
|
|
129
|
+
from the existing `{ "kind": "partition", ... }` obligation encoding.
|
|
130
|
+
|
|
131
|
+
CLI `suggest` infers this criterion from a valid, checked coverage report; its
|
|
132
|
+
command continues to reject `--criterion`. Suggested category assignments
|
|
133
|
+
replace each partition object with a concrete integer at that partition's
|
|
134
|
+
lower endpoint. That value is a representative call input only. The output
|
|
135
|
+
uses `strategy: "greedy-partition-pairwise-v1"`; it supplies no expected
|
|
136
|
+
result or behavior rule.
|
|
137
|
+
|
|
138
|
+
Finite string support is feature-specific, not guaranteed by a `0.1.x` version number. Check a fresh plan from the installed build for typed string obligations before relying on the kind. A finite string declaration enumerates scenario values and does not represent the full String domain. Only exact String instances with valid UTF-8 or US-ASCII encoding are supported; US-ASCII is canonicalized to UTF-8. Empty strings, whitespace, Unicode, and embedded NUL are valid text. Invalid byte sequences, other encodings, and String subclasses are rejected. No trimming, case folding, or Unicode normalization is applied. Declaration values and observed values are copied into frozen SpecProof-owned snapshots. The entry snapshot is used for matching even if the method mutates its argument, and direct Coverage observation also owns retained strings. A bounded outside-domain witness count does not bound witness bytes: a retained string witness is the full immutable string.
|
|
139
|
+
|
|
140
|
+
For a target also selected from RBS, DSL refinements must name parameters in
|
|
141
|
+
that selected signature. An extra DSL input produces configuration error
|
|
142
|
+
`rbs_input_mismatch`, with its DSL source and the RBS location in the message;
|
|
143
|
+
check/plan exit 2 without a successful partial plan. Unselected DSL-only targets
|
|
144
|
+
remain valid. RBS string literals, finite unions of supported string literals,
|
|
145
|
+
and optional string-literal forms import exact finite string scenarios without
|
|
146
|
+
duplicate DSL inputs. Same-spelling strings and symbols remain distinct.
|
|
147
|
+
Unrestricted `String` and selected unions with unsupported members fail
|
|
148
|
+
atomically. RBS return types are ignored; this checks input declaration names,
|
|
149
|
+
not Ruby arity/order or output conformance.
|
|
150
|
+
|
|
151
|
+
Finding witnesses use the same tagged primitive values, including the string kind above. Integer gap and containment witnesses are an array of two tagged integers denoting inclusive lower and upper bounds. Overlap witnesses are one tagged primitive. An empty-domain witness is either an empty finite array or the normalized, inverted integer bounds. For example, a gap from 65 through 120 is `[{"kind":"integer","value":"65"},{"kind":"integer","value":"120"}]`. Human reports quote and escape string values; other values retain ordinary Ruby notation.
|
|
152
|
+
|
|
153
|
+
`specproof skills export --to DIRECTORY --format json` results use `kind: "skills_export"` and add the expanded `destination` plus sorted `exported` and `unchanged` path arrays. The exporter requires an explicit destination and does not modify client or global configuration. A conflict is reported under `errors` before new files are copied.
|
|
154
|
+
|
|
155
|
+
Exit status is `0` for a successful command, `1` for semantic findings, and `2` for malformed configuration, unsupported options, usage errors, or I/O failures. A plan with semantic findings contains no obligations and returns status `1`.
|
|
156
|
+
|
|
157
|
+
## Minitest coverage
|
|
158
|
+
|
|
159
|
+
The optional adapter writes a JSON file with `kind: "coverage"` and prints a summary containing the same `run_id`. A successful report is evidence for that run only. Its contract digest does not fingerprint the implementation or tests; obtain a new run after either changes.
|
|
160
|
+
|
|
161
|
+
| Field | Meaning |
|
|
162
|
+
| --- | --- |
|
|
163
|
+
| `contract_digest`, `criterion` | Current normalized specification and selected input criterion (`input-basics-v1`, `pairwise-finite-v1`, or `pairwise-partitions-v1`); compare with a current plan |
|
|
164
|
+
| `scope.completeness_scope`, `scope.input_scope` | Declared input scope: analyzed parameter names plus explicitly excluded names and their reasons/sources |
|
|
165
|
+
| `run_id` | Fresh UUID for the invocation, also printed in its summary |
|
|
166
|
+
| `completion` | `running`, `completed`, or `incomplete` |
|
|
167
|
+
| `integrity` | Whether collection and reporting completed without a known integrity failure |
|
|
168
|
+
| `tests_passed` | Original Minitest result; `null` before a result or when the runner aborts |
|
|
169
|
+
| `obligations` | Planned obligations plus `status` and integer `hits` counters |
|
|
170
|
+
| `missing`, `exercised` | Projections of the corresponding obligation rows |
|
|
171
|
+
| `exercised_count`, `missing_count`, `unknown_count`, `not_observed_count`, `obligation_count` | Counts over the fixed planned denominator |
|
|
172
|
+
| `completion_percent` | Percentage of exercised obligations for an intact observed run, including partial coverage; `null` when collection is incomplete or no target call was observed |
|
|
173
|
+
| `outside_domain` | Per-target count and at most one typed primitive tuple witness (including string values), or an `invalid_tuple` marker; these calls fulfill no valid-input obligations |
|
|
174
|
+
| `outside_combinations`, `excluded_pairs` | Jointly forbidden runtime tuple evidence and plan pairs with no allowed full completion; excluded pairs are outside the denominator |
|
|
175
|
+
| `incomplete_reasons`, `errors` | Bounded diagnostics explaining missing integrity or unsupported collection |
|
|
176
|
+
| `evidence_mode`, `replay` | `obligation_hits` and `false`; no full event trace is retained |
|
|
177
|
+
|
|
178
|
+
An obligation is `exercised` when a valid analyzed-input tuple supplied its candidate value or category; `not_observed` when collection was intact but no such tuple was seen; or `unknown` when an integrity limitation prevents concluding absence. For partition rows, an observed integer exercises a category when it falls within the category's inclusive bounds. Positive hits remain positive after an integrity failure, but unhit affected obligations become unknown. The `missing` array contains only `not_observed` rows. Every analyzed input in a call must satisfy its domain before any candidate from that call counts. Explicitly excluded values are outside the tuple and are never read by the collector; their values and behavior are not covered. An analyzed non-primitive value still makes collection incomplete. The collector does not silently drop undeclared parameter fields to make a tuple count: declaration names must account for the selected target's complete named runtime signature, split only between analyzed and explicitly excluded inputs.
|
|
179
|
+
|
|
180
|
+
Runtime `scope.targets` lists every target in the composed contract, and runtime `scope.declaration_sources` records the supplied declaration inputs as `{ "specifications": [...], "rbs": [...], "targets": [...] }`: canonical DSL and RBS paths, plus explicitly selected RBS targets. These runtime-only invocation fields do not attest that expected test files were loaded. Runtime scope also includes the shared `completeness_scope` and `input_scope` fields described above, plus the Ruby engine, process ID, run thread identity, run arguments, recognized inclusion `filter` and `exclude` patterns, `window: "minitest_run"`, and `method_identity: "direct_owner_definition"`. It records `input_semantics: "effective_bound_values"` and `argument_presence: "not_observed"`. Inspect the original arguments as well as the pattern summaries before treating the invocation as the intended full suite; patterns do not attest which test files the caller loaded. Reports apply to the current process, selected thread, and Minitest run window. They do not merge child-process or child-thread activity or identify individual tests.
|
|
181
|
+
|
|
182
|
+
`SpecProof::Minitest.install!` accepts the same optional `rbs:` and `targets:` inputs as `SpecProof.load`. A RBS-only setup may pass `specifications: []` when it supplies both a non-empty RBS path list and selected targets. File, parser, and type validation happens during run preparation so failures produce an incomplete report while the original Minitest result remains available in `tests_passed`. Empty RBS options preserve DSL-only behavior.
|
|
183
|
+
|
|
184
|
+
Named required and optional positional/keyword parameters are supported when every parameter is either an analyzed `input` declaration or an explicit `exclude_input` with a reason, and at least one parameter remains analyzed. Entry observations contain effective analyzed values after Ruby applies defaults. Omitted and explicitly supplied equal values exercise the same obligation; neither a hit nor its count establishes argument omission. The collector does not evaluate defaults, retain call-site argument presence, or add omission-specific obligations. Defaults that raise before entry yield no target entry evidence and preserve Ruby's exception. Defaults calling another selected method can contribute that other method's own entry observations within scope. A non-primitive analyzed value makes collection incomplete; primitive analyzed values outside a declared domain invalidate the entire analyzed tuple. Excluded values are not read. Rest, keyword-rest, forwarding, and explicit block parameters remain unsupported.
|
|
185
|
+
|
|
186
|
+
Method identity is the selected owner's directly defined public Ruby instance or singleton method, resolved by canonical name before observation begins. Calls through aliases of that same definition count. A subclass override contributes to an instance target only when it executes the selected definition through `super`; a distinct override does not count merely because its method name matches. Aliased or inherited target declarations, unsupported signatures, and unresolved/autoloaded target owners fail binding. Two target names resolving to the same owner and method fail with `duplicate_binding`. Observed redefinition invalidates collection. Removing, rebinding, or replacing the selected constant path with an autoload at the end of collection produces `target_binding_changed`; previously observed hits remain, and unobserved obligations become unknown. Constant bindings must remain stable during the run; transient changes restored before final inspection are outside this guarantee. Supported runtime compatibility is CRuby 3.3, 3.4, and 4.0 with Minitest >= 5.26.2 and < 5.28.0, requiring matching package/runtime major and minor versions.
|
|
187
|
+
|
|
188
|
+
Before collection, the adapter atomically replaces any previous report with a marker for the new run, `completion: "running"`, `integrity: false`, and `tests_passed: null`. Finalization replaces it with the new run's final report. An interrupted run cannot leave an old successful report presented as current evidence. A report-write failure fails the adapter run and prints a diagnostic; a leftover running marker remains insufficient evidence.
|
|
189
|
+
|
|
190
|
+
Missing input obligations are informational by default. Strict missing mode can make the process fail even when `tests_passed` is true, because that field preserves the original assertion result. Assertion failures remain failures in either mode; collector/configuration integrity failures also fail the invocation. Input hits and assertion results are separate evidence: 100% input coverage does not establish behavioral correctness.
|
|
191
|
+
|
|
192
|
+
## Pairwise suggestions
|
|
193
|
+
|
|
194
|
+
`specproof suggest PATH --report FILE --format json` reads exactly one
|
|
195
|
+
completed coverage report and infers its criterion from that checked report.
|
|
196
|
+
It uses `greedy-pairwise-v1` for `pairwise-finite-v1` and
|
|
197
|
+
`greedy-partition-pairwise-v1` for `pairwise-partitions-v1`. It accepts the same optional `--rbs FILE` and
|
|
198
|
+
`--target TARGET` selectors as the other declaration commands. `--criterion`
|
|
199
|
+
is rejected because suggestions are defined only for pairwise reports.
|
|
200
|
+
Suggestions replan the supplied current declarations and validate the report
|
|
201
|
+
against the current contract digest, exact typed obligation denominator,
|
|
202
|
+
declared input scope, and consistent counters and projections. The source
|
|
203
|
+
report must have schema 1, kind `coverage`, completed and intact collection,
|
|
204
|
+
empty errors and incomplete reasons, zero unknown observations, and a Boolean
|
|
205
|
+
`tests_passed`. A false `tests_passed` value is accepted and preserved; it
|
|
206
|
+
records the original Minitest assertion result independently of coverage.
|
|
207
|
+
Filtered reports are accepted for their recorded scope. The output preserves
|
|
208
|
+
`source_run_id`, `source_tests_passed`, and `source_scope`; this does not make a
|
|
209
|
+
filtered run a full-suite claim.
|
|
210
|
+
|
|
211
|
+
The JSON suggestion document has schema 1, kind `suggestions`, no errors,
|
|
212
|
+
and records the criterion and strategy selected from its source report, plus
|
|
213
|
+
`assignment_scope: "combination_group"` and
|
|
214
|
+
`minimality: "not_guaranteed"`. It includes the current `contract_digest`,
|
|
215
|
+
source provenance and scope, and `missing_obligation_count`,
|
|
216
|
+
`covered_obligation_count`, and `suggestion_count`. Each suggestion row has a
|
|
217
|
+
stable `id`, `target`, a full typed `inputs` assignment for its combination
|
|
218
|
+
group, `covers` containing only newly covered missing obligation IDs,
|
|
219
|
+
`unassigned_inputs`, and source locations. The union of `covers` is exactly
|
|
220
|
+
the source report's missing set. Inputs outside a combination group and
|
|
221
|
+
explicitly excluded inputs appear in `unassigned_inputs`; the caller must
|
|
222
|
+
supply them. Every analyzed runtime input, including those names, must still
|
|
223
|
+
be valid for a call to count.
|
|
224
|
+
|
|
225
|
+
For partition rows, source obligations contain the typed category name and
|
|
226
|
+
inclusive bounds; suggestion `inputs` instead contain the lower-endpoint
|
|
227
|
+
integer representative. Do not interpret a category hit as a boundary test or
|
|
228
|
+
as evidence that all values in the category share behavior.
|
|
229
|
+
|
|
230
|
+
Suggestions are unexecuted test inputs and contain no expected outputs. Use an
|
|
231
|
+
independent requirement as the assertion oracle, and never derive expected
|
|
232
|
+
values from the implementation or the suggestions. Do not copy a candidate
|
|
233
|
+
into a test without adding a meaningful assertion. A passing pairwise report
|
|
234
|
+
or a union of suggested pairs is evidence only for the declared pair or
|
|
235
|
+
category obligations and assertions; it is not exhaustive full-tuple coverage.
|
|
236
|
+
The contract digest does not fingerprint
|
|
237
|
+
implementation or test bytes, so rerun the intended test suite in a fresh
|
|
238
|
+
process after either changes and inspect its new run ID and test result.
|
|
239
|
+
|
|
240
|
+
Suggestion generation is deterministic but greedy; it does not promise a
|
|
241
|
+
minimum number of assignments. It rejects work exceeding 10,000 cumulative
|
|
242
|
+
full assignments, 10,000 cumulative possible pair rows, 1,000,000 candidate
|
|
243
|
+
to missing-pair memberships, or a 32 MiB coverage report. It does not sample,
|
|
244
|
+
truncate, or emit partial results. CLI report reads are bounded to that size.
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
# An input gap that hides a boundary defect
|
|
2
|
+
|
|
3
|
+
For this demonstration the accepted domain is integer ages `0..120`, and the
|
|
4
|
+
intended rule is that an age of **18 or greater is adult**. That independent
|
|
5
|
+
rule supplies the assertion oracle. The input specification partitions ages
|
|
6
|
+
into minor (`0...18`) and adult (`18..120`); it does not declare return values.
|
|
7
|
+
|
|
8
|
+
Run these commands from the gem root with its development bundle and a
|
|
9
|
+
supported Minitest installation (the development bundle selects 5.26.2):
|
|
10
|
+
|
|
11
|
+
```sh
|
|
12
|
+
bundle exec specproof check examples/age_policy/specification.rb --format json
|
|
13
|
+
bundle exec specproof plan examples/age_policy/specification.rb --format json
|
|
14
|
+
bundle exec ruby examples/age_policy/run.rb missing
|
|
15
|
+
bundle exec ruby examples/age_policy/run.rb regression
|
|
16
|
+
bundle exec ruby examples/age_policy/run.rb correct
|
|
17
|
+
```
|
|
18
|
+
|
|
19
|
+
The plan proposes `0`, `1`, `17`, `18`, `119`, and `120`. The `missing` mode
|
|
20
|
+
runs the existing five non-boundary assertions against the faulty `age > 18`
|
|
21
|
+
implementation. The tests pass; the report identifies age 18 as unobserved.
|
|
22
|
+
|
|
23
|
+
The `regression` mode adds the explicit intended assertion for age 18. This
|
|
24
|
+
command intentionally exits unsuccessfully: its assertion fails against the
|
|
25
|
+
same faulty implementation. Age 18 is now exercised even though the behavior
|
|
26
|
+
is wrong, illustrating why input coverage and correctness are separate.
|
|
27
|
+
|
|
28
|
+
The `correct` mode runs all six assertions against `age >= 18`. The tests
|
|
29
|
+
pass and all six obligations are exercised. These are separate subprocesses,
|
|
30
|
+
so their alternative `AgePolicy` definitions never coexist in one run.
|
|
31
|
+
|
|
32
|
+
Each mode writes a fresh report to `tmp/age-policy-MODE.json`. An optional
|
|
33
|
+
second argument selects another report path. Match its `run_id` with the
|
|
34
|
+
printed summary and compare its contract digest with the current plan.
|
|
35
|
+
`SPECPROOF_STRICT=1` enables missing-obligation failure for this example;
|
|
36
|
+
it leaves the underlying assertion result visible in `tests_passed`.
|
|
37
|
+
|
|
38
|
+
The deterministic acceptance tests in `test/workflow_test.rb` execute this
|
|
39
|
+
same runner, including the intentionally failing mode, as subprocesses.
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "minitest/autorun"
|
|
4
|
+
require_relative "../lib/age_policy"
|
|
5
|
+
|
|
6
|
+
# Shows the faulty policy passing its non-boundary examples.
|
|
7
|
+
class AgePolicyExamplesTest < Minitest::Test
|
|
8
|
+
def test_adult_examples_without_the_boundary
|
|
9
|
+
policy = AgePolicy.new
|
|
10
|
+
|
|
11
|
+
refute policy.adult?(0)
|
|
12
|
+
refute policy.adult?(1)
|
|
13
|
+
refute policy.adult?(17)
|
|
14
|
+
assert policy.adult?(119)
|
|
15
|
+
assert policy.adult?(120)
|
|
16
|
+
end
|
|
17
|
+
end
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "minitest/autorun"
|
|
4
|
+
require_relative "../lib/age_policy"
|
|
5
|
+
|
|
6
|
+
# Demonstrates how the explicit age boundary assertion catches the defect.
|
|
7
|
+
class AgePolicyTest < Minitest::Test
|
|
8
|
+
def test_adult_examples_without_the_boundary
|
|
9
|
+
policy = AgePolicy.new
|
|
10
|
+
|
|
11
|
+
refute policy.adult?(0)
|
|
12
|
+
refute policy.adult?(1)
|
|
13
|
+
refute policy.adult?(17)
|
|
14
|
+
assert policy.adult?(119)
|
|
15
|
+
assert policy.adult?(120)
|
|
16
|
+
end
|
|
17
|
+
|
|
18
|
+
def test_explicit_boundary_requirement
|
|
19
|
+
assert AgePolicy.new.adult?(18)
|
|
20
|
+
end
|
|
21
|
+
end
|