specproof 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/CHANGELOG.md +10 -0
- data/LICENSE.txt +21 -0
- data/README.md +283 -0
- data/docs/agent-skills.md +60 -0
- data/docs/benchmark.md +24 -0
- data/docs/constrained-partition-pairs-results.md +44 -0
- data/docs/finite-strings-results.md +75 -0
- data/docs/implementation-status.md +65 -0
- data/docs/input-selection-results.md +123 -0
- data/docs/lint-cleanup-plan.md +12 -0
- data/docs/method-targets-results.md +68 -0
- data/docs/minitest-compatibility.md +33 -0
- data/docs/optional-arguments-plan.md +25 -0
- data/docs/optional-arguments-results.md +45 -0
- data/docs/pairwise-partitions-results.md +53 -0
- data/docs/pairwise-results.md +77 -0
- data/docs/pairwise-suggestions-results.md +104 -0
- data/docs/rbs-implementation-status.md +115 -0
- data/docs/rbs-string-literals-results.md +58 -0
- data/docs/real-project-pilot.md +84 -0
- data/docs/report-format.md +244 -0
- data/examples/age_policy/README.md +39 -0
- data/examples/age_policy/faulty/lib/age_policy.rb +9 -0
- data/examples/age_policy/faulty/test/age_policy_examples_test.rb +17 -0
- data/examples/age_policy/faulty/test/age_policy_test.rb +21 -0
- data/examples/age_policy/lib/age_policy.rb +8 -0
- data/examples/age_policy/run.rb +23 -0
- data/examples/age_policy/specification.rb +8 -0
- data/examples/age_policy/test/age_policy_test.rb +18 -0
- data/examples/constrained_partition_pairs/README.md +40 -0
- data/examples/constrained_partition_pairs/lib/account_eligibility.rb +8 -0
- data/examples/constrained_partition_pairs/requirements.md +11 -0
- data/examples/constrained_partition_pairs/run.rb +22 -0
- data/examples/constrained_partition_pairs/specification.rb +17 -0
- data/examples/constrained_partition_pairs/test_account_eligibility.rb +34 -0
- data/examples/input_selection/README.md +22 -0
- data/examples/input_selection/delivery_policy.rb +11 -0
- data/examples/input_selection/requirements.md +9 -0
- data/examples/input_selection/specification.rb +8 -0
- data/examples/input_selection/test_delivery_policy.rb +38 -0
- data/examples/method_targets/README.md +30 -0
- data/examples/method_targets/lib/delivery_defaults.rb +8 -0
- data/examples/method_targets/lib/delivery_options.rb +12 -0
- data/examples/method_targets/requirements.md +20 -0
- data/examples/method_targets/specification_objects.rb +17 -0
- data/examples/method_targets/specification_strings.rb +15 -0
- data/examples/pairwise/README.md +46 -0
- data/examples/pairwise/constrained_requirements.md +6 -0
- data/examples/pairwise/constrained_specification.rb +11 -0
- data/examples/pairwise/corrected/lib/delivery_options.rb +11 -0
- data/examples/pairwise/corrected/test/delivery_options_test.rb +28 -0
- data/examples/pairwise/faulty/lib/delivery_options.rb +11 -0
- data/examples/pairwise/faulty/test/gap_delivery_options_test.rb +10 -0
- data/examples/pairwise/faulty/test/initial_delivery_options_test.rb +23 -0
- data/examples/pairwise/requirements.md +11 -0
- data/examples/pairwise/run.rb +36 -0
- data/examples/pairwise/specification.rb +9 -0
- data/examples/pairwise/three_boolean/lib/boolean_batch.rb +8 -0
- data/examples/pairwise/three_boolean/test/boolean_batch_test.rb +28 -0
- data/examples/pairwise/three_boolean_specification.rb +10 -0
- data/examples/pairwise_partitions/README.md +34 -0
- data/examples/pairwise_partitions/lib/age_policy.rb +8 -0
- data/examples/pairwise_partitions/requirements.md +8 -0
- data/examples/pairwise_partitions/run.rb +22 -0
- data/examples/pairwise_partitions/specification.rb +13 -0
- data/examples/pairwise_partitions/test_age_policy.rb +29 -0
- data/examples/pairwise_suggestions/README.md +48 -0
- data/examples/pairwise_suggestions/run.rb +55 -0
- data/examples/pairwise_suggestions/three_boolean/lib/suggested_boolean_batch.rb +8 -0
- data/examples/pairwise_suggestions/three_boolean/requirements.md +7 -0
- data/examples/pairwise_suggestions/three_boolean/specification.rb +11 -0
- data/examples/pairwise_suggestions/three_boolean/test/baseline_test.rb +9 -0
- data/examples/pairwise_suggestions/three_boolean/test/complete_test.rb +22 -0
- data/examples/partition_gap.rb +8 -0
- data/examples/rbs_registration/README.md +58 -0
- data/examples/rbs_registration/implementations/age_gt/registration.rb +8 -0
- data/examples/rbs_registration/implementations/age_gte/registration.rb +8 -0
- data/examples/rbs_registration/requirements.md +9 -0
- data/examples/rbs_registration/sig/registration.rbs +3 -0
- data/examples/rbs_registration/specification.rb +10 -0
- data/examples/rbs_registration/test_registration.rb +43 -0
- data/examples/rbs_string_literals/README.md +34 -0
- data/examples/rbs_string_literals/lib/delivery_options.rb +9 -0
- data/examples/rbs_string_literals/requirements.md +11 -0
- data/examples/rbs_string_literals/sig/delivery_options.rbs +3 -0
- data/examples/rbs_string_literals/specification.rb +7 -0
- data/examples/rbs_string_literals/test_delivery_options.rb +46 -0
- data/examples/shipping.rb +12 -0
- data/exe/specproof +7 -0
- data/lib/specproof/checker.rb +144 -0
- data/lib/specproof/cli.rb +292 -0
- data/lib/specproof/collector.rb +304 -0
- data/lib/specproof/combinations.rb +226 -0
- data/lib/specproof/composer.rb +97 -0
- data/lib/specproof/coverage.rb +229 -0
- data/lib/specproof/coverage_report.rb +334 -0
- data/lib/specproof/domains.rb +124 -0
- data/lib/specproof/loader.rb +285 -0
- data/lib/specproof/minitest.rb +443 -0
- data/lib/specproof/model.rb +198 -0
- data/lib/specproof/partition_space.rb +268 -0
- data/lib/specproof/planner.rb +125 -0
- data/lib/specproof/report.rb +248 -0
- data/lib/specproof/skills.rb +188 -0
- data/lib/specproof/sources/rbs.rb +137 -0
- data/lib/specproof/sources/rbs_projector.rb +268 -0
- data/lib/specproof/suggestion_report.rb +159 -0
- data/lib/specproof/suggestions.rb +352 -0
- data/lib/specproof/target_name.rb +100 -0
- data/lib/specproof/type_constraints.rb +76 -0
- data/lib/specproof/version.rb +7 -0
- data/lib/specproof.rb +100 -0
- data/sig/specproof.rbs +222 -0
- data/skills/specproof-specify/SKILL.md +282 -0
- data/skills/specproof-specify/references/usage.md +265 -0
- data/skills/specproof-test-gaps/SKILL.md +214 -0
- data/skills/specproof-test-gaps/references/usage.md +231 -0
- metadata +165 -0
|
@@ -0,0 +1,229 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module SpecProof
|
|
4
|
+
# Tracks whether planned input obligations have been observed at runtime.
|
|
5
|
+
# It keeps aggregate counters and a small set of invalid-input witnesses,
|
|
6
|
+
# never an event trace.
|
|
7
|
+
class Coverage
|
|
8
|
+
WITNESS_LIMIT = 1
|
|
9
|
+
REASON_LIMIT = 20
|
|
10
|
+
|
|
11
|
+
def initialize(obligations:, specifications:)
|
|
12
|
+
@specifications = specifications.to_h { |specification| [specification.target, specification] }.freeze
|
|
13
|
+
@obligations = obligations.dup.freeze
|
|
14
|
+
@hits = @obligations.to_h { |obligation| [obligation.id, 0] }
|
|
15
|
+
@spaces = specifications.filter_map do |specification|
|
|
16
|
+
next unless specification.respond_to?(:combination_group) && specification.combination_group
|
|
17
|
+
|
|
18
|
+
space = PartitionSpace.for_specification(specification)
|
|
19
|
+
[specification.target, space]
|
|
20
|
+
end.to_h.freeze
|
|
21
|
+
@outside_counts = Hash.new(0)
|
|
22
|
+
@outside_witnesses = Hash.new { |hash, target| hash[target] = [] }
|
|
23
|
+
@outside_combination_counts = Hash.new(0)
|
|
24
|
+
@outside_combination_evidence = {}
|
|
25
|
+
@incomplete_reasons = Hash.new { |hash, target| hash[target] = [] }
|
|
26
|
+
@observed_targets = {}
|
|
27
|
+
@incomplete_targets = {}
|
|
28
|
+
end
|
|
29
|
+
|
|
30
|
+
# This command mutates coverage; its boolean reports acceptance, not state.
|
|
31
|
+
def observe(target:, arguments:) # rubocop:disable Naming/PredicateMethod
|
|
32
|
+
target = target.to_s
|
|
33
|
+
specification = @specifications[target]
|
|
34
|
+
return false unless specification
|
|
35
|
+
|
|
36
|
+
@observed_targets[target] = true
|
|
37
|
+
tuple = primitive_tuple(arguments)
|
|
38
|
+
unless valid_tuple?(specification, tuple)
|
|
39
|
+
record_outside(target, tuple)
|
|
40
|
+
return false
|
|
41
|
+
end
|
|
42
|
+
space = @spaces[target]
|
|
43
|
+
if space && !space.allowed?(tuple)
|
|
44
|
+
record_outside(target, tuple)
|
|
45
|
+
record_outside_combination(target, tuple, space.sources)
|
|
46
|
+
return false
|
|
47
|
+
end
|
|
48
|
+
|
|
49
|
+
@obligations.each do |obligation|
|
|
50
|
+
next unless obligation.target == target
|
|
51
|
+
|
|
52
|
+
if obligation.respond_to?(:inputs) && obligation.inputs
|
|
53
|
+
next unless obligation.inputs.all? do |name, value|
|
|
54
|
+
actual = tuple.fetch(name)
|
|
55
|
+
value.is_a?(PartitionLevel) ? value.include?(actual) : actual == value
|
|
56
|
+
end
|
|
57
|
+
else
|
|
58
|
+
next unless tuple.fetch(obligation.input) == obligation.value
|
|
59
|
+
end
|
|
60
|
+
|
|
61
|
+
@hits[obligation.id] += 1
|
|
62
|
+
end
|
|
63
|
+
true
|
|
64
|
+
end
|
|
65
|
+
|
|
66
|
+
def mark_incomplete(target:, reason:)
|
|
67
|
+
target = target.to_s
|
|
68
|
+
@incomplete_targets[target] = true
|
|
69
|
+
reasons = @incomplete_reasons[target]
|
|
70
|
+
value = primitive_string(reason)
|
|
71
|
+
reasons << value if reasons.length < REASON_LIMIT && !reasons.include?(value)
|
|
72
|
+
self
|
|
73
|
+
end
|
|
74
|
+
|
|
75
|
+
def missing
|
|
76
|
+
to_report.fetch(:missing)
|
|
77
|
+
end
|
|
78
|
+
|
|
79
|
+
def exercised
|
|
80
|
+
to_report.fetch(:exercised)
|
|
81
|
+
end
|
|
82
|
+
|
|
83
|
+
# Returns a compact projection suitable for wrapping in Report.envelope.
|
|
84
|
+
def to_report
|
|
85
|
+
projection = base_projection
|
|
86
|
+
projection.merge!(combination_projection) unless @spaces.empty?
|
|
87
|
+
projection
|
|
88
|
+
end
|
|
89
|
+
|
|
90
|
+
private
|
|
91
|
+
|
|
92
|
+
def base_projection
|
|
93
|
+
rows = obligation_rows
|
|
94
|
+
exercised_rows = rows.select { |row| row.fetch(:status) == "exercised" }
|
|
95
|
+
missing_rows = rows.select { |row| row.fetch(:status) == "not_observed" }
|
|
96
|
+
unknown_count = rows.count { |row| row.fetch(:status) == "unknown" }
|
|
97
|
+
not_observed_count = missing_rows.length
|
|
98
|
+
exercised = exercised_rows.length
|
|
99
|
+
incomplete_reasons = @incomplete_reasons.values.flatten.uniq
|
|
100
|
+
complete = @incomplete_targets.none?
|
|
101
|
+
observed = !@observed_targets.empty?
|
|
102
|
+
{
|
|
103
|
+
obligations: rows,
|
|
104
|
+
missing: missing_rows,
|
|
105
|
+
exercised: exercised_rows,
|
|
106
|
+
exercised_count: exercised,
|
|
107
|
+
obligation_count: rows.length,
|
|
108
|
+
missing_count: missing_rows.length,
|
|
109
|
+
unknown_count: unknown_count,
|
|
110
|
+
not_observed_count: not_observed_count,
|
|
111
|
+
completion_percent: completion_percent(rows, complete, observed, exercised),
|
|
112
|
+
status: if observed
|
|
113
|
+
(complete ? "complete" : "incomplete")
|
|
114
|
+
else
|
|
115
|
+
"not_observed"
|
|
116
|
+
end,
|
|
117
|
+
completeness: complete ? "complete" : "incomplete",
|
|
118
|
+
outside_domain: outside_domain_report,
|
|
119
|
+
incomplete_reasons: incomplete_reasons,
|
|
120
|
+
evidence_mode: "obligation_hits",
|
|
121
|
+
replay: false
|
|
122
|
+
}
|
|
123
|
+
end
|
|
124
|
+
|
|
125
|
+
def obligation_rows
|
|
126
|
+
@obligations.map do |obligation|
|
|
127
|
+
hits = @hits.fetch(obligation.id)
|
|
128
|
+
state = obligation_state(hits, obligation.target)
|
|
129
|
+
Report.obligation(obligation).merge(status: state, hits: hits)
|
|
130
|
+
end
|
|
131
|
+
end
|
|
132
|
+
|
|
133
|
+
def obligation_state(hits, target)
|
|
134
|
+
return "exercised" if hits.positive?
|
|
135
|
+
return "unknown" if @incomplete_targets[target]
|
|
136
|
+
|
|
137
|
+
"not_observed"
|
|
138
|
+
end
|
|
139
|
+
|
|
140
|
+
def outside_domain_report
|
|
141
|
+
@outside_counts.to_h do |target, count|
|
|
142
|
+
witness = @outside_witnesses.fetch(target).first
|
|
143
|
+
[target, { count: count, witness: typed_tuple(witness) }]
|
|
144
|
+
end
|
|
145
|
+
end
|
|
146
|
+
|
|
147
|
+
def combination_projection
|
|
148
|
+
{
|
|
149
|
+
outside_combinations: outside_combinations_report,
|
|
150
|
+
excluded_pairs: excluded_pairs_report
|
|
151
|
+
}
|
|
152
|
+
end
|
|
153
|
+
|
|
154
|
+
def outside_combinations_report
|
|
155
|
+
@outside_combination_counts.to_h do |target, count|
|
|
156
|
+
evidence = @outside_combination_evidence.fetch(target)
|
|
157
|
+
[target, { count: count, witness: typed_tuple(evidence.fetch(:witness)),
|
|
158
|
+
sources: evidence.fetch(:sources).map { |item| Report.source(item) } }]
|
|
159
|
+
end
|
|
160
|
+
end
|
|
161
|
+
|
|
162
|
+
def excluded_pairs_report
|
|
163
|
+
@spaces.flat_map do |target, space|
|
|
164
|
+
space.excluded_pairs.map do |inputs|
|
|
165
|
+
{ target: target, inputs: typed_inputs(inputs), reason: "no_allowed_completion",
|
|
166
|
+
sources: space.sources.map { |item| Report.source(item) } }
|
|
167
|
+
end
|
|
168
|
+
end
|
|
169
|
+
end
|
|
170
|
+
|
|
171
|
+
def completion_percent(rows, complete, observed, exercised)
|
|
172
|
+
return if rows.empty? || !complete || !observed
|
|
173
|
+
|
|
174
|
+
((exercised * 100.0) / rows.length).round(2)
|
|
175
|
+
end
|
|
176
|
+
|
|
177
|
+
def valid_tuple?(specification, tuple)
|
|
178
|
+
return false unless tuple
|
|
179
|
+
|
|
180
|
+
expected = specification.inputs.keys
|
|
181
|
+
return false unless tuple.keys.sort_by(&:to_s) == expected.sort_by(&:to_s)
|
|
182
|
+
|
|
183
|
+
specification.inputs.all? do |name, input|
|
|
184
|
+
Domains.include?(input.domain, tuple.fetch(name))
|
|
185
|
+
end
|
|
186
|
+
end
|
|
187
|
+
|
|
188
|
+
def record_outside_combination(target, tuple, sources)
|
|
189
|
+
@outside_combination_counts[target] += 1
|
|
190
|
+
@outside_combination_evidence[target] ||= { witness: tuple, sources: sources.dup.freeze }.freeze
|
|
191
|
+
end
|
|
192
|
+
|
|
193
|
+
def typed_inputs(inputs)
|
|
194
|
+
inputs.to_h { |name, value| [name.to_s, Report.value(value)] }
|
|
195
|
+
end
|
|
196
|
+
|
|
197
|
+
def primitive_tuple(arguments)
|
|
198
|
+
return nil unless arguments.is_a?(Hash)
|
|
199
|
+
|
|
200
|
+
arguments.each_with_object({}) do |(key, value), normalized|
|
|
201
|
+
return nil unless key.is_a?(Symbol)
|
|
202
|
+
|
|
203
|
+
name = key
|
|
204
|
+
return nil unless Domains.primitive?(value)
|
|
205
|
+
|
|
206
|
+
normalized[name] = Domains.supported_string?(value) ? Domains.canonical_string(value) : value
|
|
207
|
+
rescue NoMethodError
|
|
208
|
+
return nil
|
|
209
|
+
end.freeze
|
|
210
|
+
end
|
|
211
|
+
|
|
212
|
+
def record_outside(target, tuple)
|
|
213
|
+
@outside_counts[target] += 1
|
|
214
|
+
witnesses = @outside_witnesses[target]
|
|
215
|
+
witnesses << tuple if witnesses.length < WITNESS_LIMIT && !witnesses.include?(tuple)
|
|
216
|
+
end
|
|
217
|
+
|
|
218
|
+
def typed_tuple(tuple)
|
|
219
|
+
return { kind: "invalid_tuple" } unless tuple
|
|
220
|
+
|
|
221
|
+
tuple.to_h { |key, value| [key.to_s, Report.value(value)] }
|
|
222
|
+
end
|
|
223
|
+
|
|
224
|
+
def primitive_string(value)
|
|
225
|
+
text = value.is_a?(String) || value.is_a?(Symbol) ? value.to_s : "incomplete collection"
|
|
226
|
+
text.encode("UTF-8", invalid: :replace, undef: :replace, replace: "?").byteslice(0, 240).to_s.freeze
|
|
227
|
+
end
|
|
228
|
+
end
|
|
229
|
+
end
|
|
@@ -0,0 +1,334 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module SpecProof
|
|
4
|
+
# Supplies stable validation failures to the report reader.
|
|
5
|
+
module CoverageReportErrors
|
|
6
|
+
private
|
|
7
|
+
|
|
8
|
+
def unsupported!(message) = raise ConfigurationError.new(message, code: "unsupported_coverage_report")
|
|
9
|
+
|
|
10
|
+
def incomplete!(message) = raise ConfigurationError.new(message, code: "incomplete_coverage_report")
|
|
11
|
+
|
|
12
|
+
def mismatch!(message) = raise ConfigurationError.new(message, code: "coverage_contract_mismatch")
|
|
13
|
+
|
|
14
|
+
def invalid!(message) = raise ConfigurationError.new(message, code: "invalid_coverage_report")
|
|
15
|
+
end
|
|
16
|
+
|
|
17
|
+
# Checks and owns the JSON-shaped data preserved by a validated report.
|
|
18
|
+
module CoverageReportJSON
|
|
19
|
+
private
|
|
20
|
+
|
|
21
|
+
def deep_copy_freeze(value)
|
|
22
|
+
case value
|
|
23
|
+
when Hash
|
|
24
|
+
value.to_h { |key, item| [key.dup.freeze, deep_copy_freeze(item)] }.freeze
|
|
25
|
+
when Array
|
|
26
|
+
value.map { |item| deep_copy_freeze(item) }.freeze
|
|
27
|
+
when String
|
|
28
|
+
value.dup.freeze
|
|
29
|
+
else
|
|
30
|
+
value.freeze
|
|
31
|
+
end
|
|
32
|
+
end
|
|
33
|
+
|
|
34
|
+
def json_object?(value)
|
|
35
|
+
value.is_a?(Hash) && value.keys.all?(String) &&
|
|
36
|
+
value.values.all? { |item| json_value?(item) }
|
|
37
|
+
end
|
|
38
|
+
|
|
39
|
+
def json_value?(value)
|
|
40
|
+
case value
|
|
41
|
+
when Hash then value.keys.all?(String) && value.values.all? { |item| json_value?(item) }
|
|
42
|
+
when Array then value.all? { |item| json_value?(item) }
|
|
43
|
+
when String, Integer, TrueClass, FalseClass, NilClass then true
|
|
44
|
+
when Float then value.finite?
|
|
45
|
+
else false
|
|
46
|
+
end
|
|
47
|
+
end
|
|
48
|
+
end
|
|
49
|
+
|
|
50
|
+
# Validates bounded runtime evidence against a freshly planned pairwise contract.
|
|
51
|
+
class CoverageReport
|
|
52
|
+
include CoverageReportErrors
|
|
53
|
+
include CoverageReportJSON
|
|
54
|
+
|
|
55
|
+
SCHEMA_VERSION = 1
|
|
56
|
+
KIND = "coverage"
|
|
57
|
+
CRITERION = "pairwise-finite-v1"
|
|
58
|
+
PARTITION_CRITERION = "pairwise-partitions-v1"
|
|
59
|
+
REQUIRED_SCOPE = {
|
|
60
|
+
"window" => "minitest_run",
|
|
61
|
+
"method_identity" => "direct_owner_definition",
|
|
62
|
+
"input_semantics" => "effective_bound_values",
|
|
63
|
+
"argument_presence" => "not_observed"
|
|
64
|
+
}.freeze
|
|
65
|
+
REQUIRED_FIELDS = %w[
|
|
66
|
+
schema_version kind errors contract_digest criterion run_id scope evidence_mode replay
|
|
67
|
+
obligations missing exercised exercised_count obligation_count missing_count unknown_count
|
|
68
|
+
not_observed_count completion_percent completion integrity tests_passed incomplete_reasons
|
|
69
|
+
].freeze
|
|
70
|
+
|
|
71
|
+
attr_reader :contract_digest, :missing_ids, :run_id, :tests_passed, :scope, :scope_snapshot, :criterion
|
|
72
|
+
|
|
73
|
+
def initialize(report, specifications:, obligations:, criterion: CRITERION)
|
|
74
|
+
@specifications = specifications
|
|
75
|
+
@obligations = obligations
|
|
76
|
+
@criterion = criterion.is_a?(String) ? criterion.dup.freeze : criterion
|
|
77
|
+
validate_document!(report)
|
|
78
|
+
@contract_digest, @run_id = report.values_at("contract_digest", "run_id").map { |value| value.dup.freeze }
|
|
79
|
+
@scope = @scope_snapshot = deep_copy_freeze(report.fetch("scope"))
|
|
80
|
+
@tests_passed = report.fetch("tests_passed")
|
|
81
|
+
@missing_ids = report.fetch("missing").map { |row| row.fetch("id").dup.freeze }.freeze
|
|
82
|
+
freeze
|
|
83
|
+
end
|
|
84
|
+
|
|
85
|
+
private
|
|
86
|
+
|
|
87
|
+
def validate_document!(report)
|
|
88
|
+
invalid!("coverage report must be a JSON object") unless report.is_a?(Hash) && report.keys.all?(String)
|
|
89
|
+
missing_fields = REQUIRED_FIELDS - report.keys
|
|
90
|
+
invalid!("coverage report is missing required fields: #{missing_fields.join(", ")}") unless missing_fields.empty?
|
|
91
|
+
validate_identity!(report)
|
|
92
|
+
validate_completion!(report)
|
|
93
|
+
validate_rows!(report)
|
|
94
|
+
validate_projections!(report)
|
|
95
|
+
validate_counts!(report)
|
|
96
|
+
validate_completion_percent!(report)
|
|
97
|
+
validate_scope!(report.fetch("scope"))
|
|
98
|
+
end
|
|
99
|
+
|
|
100
|
+
def validate_identity!(report)
|
|
101
|
+
unless report["schema_version"].is_a?(Integer) && report["schema_version"] == SCHEMA_VERSION &&
|
|
102
|
+
report["kind"] == KIND && [CRITERION, PARTITION_CRITERION].include?(criterion) &&
|
|
103
|
+
report["criterion"] == criterion
|
|
104
|
+
unsupported!("coverage report schema, kind, or criterion is unsupported")
|
|
105
|
+
end
|
|
106
|
+
unless report["run_id"].is_a?(String) && !report["run_id"].empty?
|
|
107
|
+
invalid!("coverage report run_id must be a non-empty string")
|
|
108
|
+
end
|
|
109
|
+
return if report["contract_digest"] == Report.digest(@specifications)
|
|
110
|
+
|
|
111
|
+
mismatch!("coverage report contract digest does not match specifications")
|
|
112
|
+
end
|
|
113
|
+
|
|
114
|
+
def validate_completion!(report)
|
|
115
|
+
invalid!("coverage report completion must be a string") unless report["completion"].is_a?(String)
|
|
116
|
+
incomplete!("coverage report is not complete") unless report["completion"] == "completed"
|
|
117
|
+
invalid!("coverage report integrity must be boolean") unless [true, false].include?(report["integrity"])
|
|
118
|
+
incomplete!("coverage report integrity is false") unless report["integrity"]
|
|
119
|
+
invalid!("coverage report errors must be an array") unless report["errors"].is_a?(Array)
|
|
120
|
+
incomplete!("coverage report contains errors") unless report["errors"].empty?
|
|
121
|
+
invalid!("coverage report incomplete_reasons must be an array of strings") unless
|
|
122
|
+
report["incomplete_reasons"].is_a?(Array) && report["incomplete_reasons"].all?(String)
|
|
123
|
+
incomplete!("coverage report contains incomplete reasons") unless report["incomplete_reasons"].empty?
|
|
124
|
+
invalid!("coverage report evidence_mode must be a string") unless report["evidence_mode"].is_a?(String)
|
|
125
|
+
invalid!("coverage report replay must be boolean") unless [true, false].include?(report["replay"])
|
|
126
|
+
unless report["evidence_mode"] == "obligation_hits" && report["replay"] == false
|
|
127
|
+
incomplete!("coverage report does not contain direct obligation-hit evidence")
|
|
128
|
+
end
|
|
129
|
+
unless report["tests_passed"].equal?(true) || report["tests_passed"].equal?(false)
|
|
130
|
+
invalid!("coverage report tests_passed must be boolean")
|
|
131
|
+
end
|
|
132
|
+
invalid!("coverage report unknown_count must be an integer") unless report["unknown_count"].is_a?(Integer)
|
|
133
|
+
return if report["unknown_count"].zero?
|
|
134
|
+
|
|
135
|
+
incomplete!("coverage report contains unknown obligations")
|
|
136
|
+
end
|
|
137
|
+
|
|
138
|
+
def validate_scope!(scope)
|
|
139
|
+
invalid!("coverage report scope must be a JSON object") unless json_object?(scope)
|
|
140
|
+
REQUIRED_SCOPE.each do |key, value|
|
|
141
|
+
mismatch!("coverage report scope #{key} does not match the adapter") unless scope[key] == value
|
|
142
|
+
end
|
|
143
|
+
unless scope.key?("arguments") && scope["arguments"].is_a?(Array) && scope["arguments"].all?(String) &&
|
|
144
|
+
scope.key?("filter") && [String, NilClass].include?(scope["filter"].class) &&
|
|
145
|
+
scope.key?("exclude") && [String, NilClass].include?(scope["exclude"].class)
|
|
146
|
+
invalid!("coverage report scope arguments, filter, or exclude has an invalid type")
|
|
147
|
+
end
|
|
148
|
+
expected_targets = @specifications.map(&:target).sort
|
|
149
|
+
unless scope["targets"].is_a?(Array) && scope["targets"].all?(String)
|
|
150
|
+
invalid!("coverage report scope targets must be strings")
|
|
151
|
+
end
|
|
152
|
+
mismatch!("coverage report targets do not match specifications") unless scope["targets"].sort == expected_targets
|
|
153
|
+
expected_scope = Report.input_scope(@specifications)
|
|
154
|
+
actual_scope = scope["input_scope"]
|
|
155
|
+
invalid!("coverage report input_scope must be an array") unless actual_scope.is_a?(Array)
|
|
156
|
+
unless scope_projection(actual_scope) == scope_projection(expected_scope[:input_scope])
|
|
157
|
+
mismatch!("coverage report analyzed input scope does not match specifications")
|
|
158
|
+
end
|
|
159
|
+
unless scope["completeness_scope"] == "analyzed_inputs"
|
|
160
|
+
mismatch!("coverage report completeness scope does not match specifications")
|
|
161
|
+
end
|
|
162
|
+
rescue NoMethodError, ArgumentError, TypeError
|
|
163
|
+
mismatch!("coverage report scope does not match specifications")
|
|
164
|
+
end
|
|
165
|
+
|
|
166
|
+
def validate_rows!(report)
|
|
167
|
+
rows = report["obligations"]
|
|
168
|
+
invalid!("coverage report obligations must be an array") unless rows.is_a?(Array)
|
|
169
|
+
mismatch!("coverage report obligation denominator does not match plan") unless rows.length == @obligations.length
|
|
170
|
+
|
|
171
|
+
expected = @obligations.to_h { |obligation| [obligation.id, obligation] }
|
|
172
|
+
mismatch!("planned obligation IDs are not unique") unless expected.length == @obligations.length
|
|
173
|
+
seen = {}
|
|
174
|
+
rows.each do |row|
|
|
175
|
+
invalid!("coverage obligation row must be an object") unless row.is_a?(Hash) && row.keys.all?(String)
|
|
176
|
+
id = row["id"]
|
|
177
|
+
invalid!("coverage obligation ID must be a string") unless id.is_a?(String) && !id.empty?
|
|
178
|
+
invalid!("coverage obligation rows contain duplicate IDs") if seen[id]
|
|
179
|
+
obligation = expected[id]
|
|
180
|
+
mismatch!("coverage obligation IDs do not match plan") unless obligation
|
|
181
|
+
validate_row!(row, obligation)
|
|
182
|
+
seen[id] = true
|
|
183
|
+
end
|
|
184
|
+
mismatch!("coverage obligation IDs do not match plan") unless seen.length == expected.length
|
|
185
|
+
end
|
|
186
|
+
|
|
187
|
+
def validate_row!(row, obligation)
|
|
188
|
+
invalid!("coverage obligation hits must be an integer") unless row["hits"].is_a?(Integer)
|
|
189
|
+
invalid!("coverage obligation target must be a string") unless row["target"].is_a?(String)
|
|
190
|
+
unless row["inputs"].is_a?(Hash) && row["inputs"].keys.all?(String)
|
|
191
|
+
invalid!("coverage obligation inputs must be an object")
|
|
192
|
+
end
|
|
193
|
+
row["inputs"].each_value do |typed|
|
|
194
|
+
invalid!("coverage obligation input value is malformed") unless valid_typed_value?(typed)
|
|
195
|
+
end
|
|
196
|
+
unless row["status"].is_a?(String) && %w[exercised not_observed unknown].include?(row["status"])
|
|
197
|
+
invalid!("coverage obligation status is invalid")
|
|
198
|
+
end
|
|
199
|
+
incomplete!("coverage report contains unknown obligations") if row["status"] == "unknown"
|
|
200
|
+
unless row["target"].eql?(obligation.target) && row["inputs"].eql?(typed_inputs(obligation.inputs))
|
|
201
|
+
mismatch!("coverage obligation contents do not match plan")
|
|
202
|
+
end
|
|
203
|
+
valid_hits = (row["status"] == "exercised" && row["hits"].positive?) ||
|
|
204
|
+
(row["status"] == "not_observed" && row["hits"].zero?)
|
|
205
|
+
invalid!("coverage obligation status and hits disagree") unless valid_hits
|
|
206
|
+
end
|
|
207
|
+
|
|
208
|
+
def validate_projections!(report)
|
|
209
|
+
rows = report.fetch("obligations")
|
|
210
|
+
{ "missing" => "not_observed", "exercised" => "exercised" }.each do |field, status|
|
|
211
|
+
projection = report[field]
|
|
212
|
+
invalid!("coverage report #{field} projection must be an array") unless projection.is_a?(Array)
|
|
213
|
+
expected = rows.select { |row| row.fetch("status") == status }
|
|
214
|
+
unless projection.length == expected.length
|
|
215
|
+
invalid!("coverage report #{field} projection has an invalid row count")
|
|
216
|
+
end
|
|
217
|
+
projection.each { |item| validate_projection_row!(item, field) }
|
|
218
|
+
invalid!("coverage report #{field} projection disagrees with obligations") unless
|
|
219
|
+
projection_signature(projection).eql?(projection_signature(expected))
|
|
220
|
+
end
|
|
221
|
+
end
|
|
222
|
+
|
|
223
|
+
def validate_projection_row!(row, field)
|
|
224
|
+
invalid!("coverage report #{field} projection contains a malformed row") unless projection_row?(row)
|
|
225
|
+
end
|
|
226
|
+
|
|
227
|
+
def projection_row?(row)
|
|
228
|
+
return false unless row.is_a?(Hash) && row.keys.all?(String)
|
|
229
|
+
|
|
230
|
+
%w[id target status].all? { |key| row[key].is_a?(String) } &&
|
|
231
|
+
row["inputs"].is_a?(Hash) && row["inputs"].keys.all?(String) &&
|
|
232
|
+
row["inputs"].values.all? { |typed| valid_typed_value?(typed) } && row["hits"].is_a?(Integer)
|
|
233
|
+
end
|
|
234
|
+
|
|
235
|
+
def valid_typed_value?(typed)
|
|
236
|
+
return false unless typed.is_a?(Hash) && typed.keys.all?(String) && typed["kind"].is_a?(String)
|
|
237
|
+
|
|
238
|
+
case typed["kind"]
|
|
239
|
+
when "nil" then typed.keys.sort == %w[kind value] && typed["value"].nil?
|
|
240
|
+
when "boolean" then typed.keys.sort == %w[kind value] && [true, false].include?(typed["value"])
|
|
241
|
+
when "integer", "symbol", "string"
|
|
242
|
+
typed.keys.sort == %w[kind value] && typed["value"].is_a?(String)
|
|
243
|
+
when "partition"
|
|
244
|
+
typed.keys.sort == %w[kind lower name upper] && typed["name"].is_a?(String) &&
|
|
245
|
+
canonical_integer_string?(typed["lower"]) && canonical_integer_string?(typed["upper"]) &&
|
|
246
|
+
Integer(typed["lower"], 10) <= Integer(typed["upper"], 10)
|
|
247
|
+
else false
|
|
248
|
+
end
|
|
249
|
+
rescue ArgumentError
|
|
250
|
+
false
|
|
251
|
+
end
|
|
252
|
+
|
|
253
|
+
def canonical_integer_string?(value)
|
|
254
|
+
value.is_a?(String) && value.match?(/\A-?(?:0|[1-9]\d*)\z/) && Integer(value, 10).to_s == value
|
|
255
|
+
end
|
|
256
|
+
|
|
257
|
+
def validate_counts!(report)
|
|
258
|
+
values = %w[exercised_count obligation_count missing_count unknown_count not_observed_count]
|
|
259
|
+
invalid!("coverage report counts must be integers") unless values.all? { |key| report[key].is_a?(Integer) }
|
|
260
|
+
exercised = report.fetch("obligations").count { |row| row.fetch("status") == "exercised" }
|
|
261
|
+
missing = report.fetch("obligations").count { |row| row.fetch("status") == "not_observed" }
|
|
262
|
+
expected = { "exercised_count" => exercised, "obligation_count" => @obligations.length,
|
|
263
|
+
"missing_count" => missing, "unknown_count" => 0, "not_observed_count" => missing }
|
|
264
|
+
invalid!("coverage report counters disagree with obligation rows") unless values.to_h do |key|
|
|
265
|
+
[key, report[key]]
|
|
266
|
+
end == expected
|
|
267
|
+
end
|
|
268
|
+
|
|
269
|
+
def validate_completion_percent!(report)
|
|
270
|
+
value = report["completion_percent"]
|
|
271
|
+
return if value.nil? && report["exercised_count"].zero?
|
|
272
|
+
|
|
273
|
+
invalid!("coverage report cannot give a percentage for an empty denominator") if report["obligation_count"].zero?
|
|
274
|
+
unless value.is_a?(Numeric) && !value.is_a?(Complex)
|
|
275
|
+
invalid!("coverage report completion_percent must be numeric or null")
|
|
276
|
+
end
|
|
277
|
+
|
|
278
|
+
expected = ((report.fetch("exercised_count") * 100.0) / report.fetch("obligation_count")).round(2)
|
|
279
|
+
invalid!("coverage report completion_percent disagrees with counters") unless value == expected
|
|
280
|
+
end
|
|
281
|
+
|
|
282
|
+
def scope_projection(scope)
|
|
283
|
+
return nil unless scope.is_a?(Array)
|
|
284
|
+
|
|
285
|
+
entries = scope.map { |item| scope_entry_projection(item) }
|
|
286
|
+
entries.sort
|
|
287
|
+
rescue ArgumentError, TypeError
|
|
288
|
+
invalid!("coverage report input scope is malformed")
|
|
289
|
+
end
|
|
290
|
+
|
|
291
|
+
def scope_entry_projection(item)
|
|
292
|
+
invalid!("coverage report input scope is malformed") unless item.is_a?(Hash)
|
|
293
|
+
|
|
294
|
+
target = item["target"] || item[:target]
|
|
295
|
+
names = item["analyzed_inputs"] || item[:analyzed_inputs]
|
|
296
|
+
excluded = item["excluded_inputs"] || item[:excluded_inputs]
|
|
297
|
+
group = item["combination_group"] || item[:combination_group]
|
|
298
|
+
unless target.is_a?(String) && names.is_a?(Array) && names.all?(String) && excluded.is_a?(Array)
|
|
299
|
+
invalid!("coverage report input scope is malformed")
|
|
300
|
+
end
|
|
301
|
+
exclusion_names = excluded.map { |entry| exclusion_name(entry) }
|
|
302
|
+
group_inputs = group && group_inputs_for_scope(group)
|
|
303
|
+
[target, names.sort, exclusion_names.sort, group_inputs&.sort]
|
|
304
|
+
end
|
|
305
|
+
|
|
306
|
+
def exclusion_name(entry)
|
|
307
|
+
name = entry.is_a?(Hash) ? entry["name"] || entry[:name] : nil
|
|
308
|
+
invalid!("coverage report excluded input scope is malformed") unless name.is_a?(String)
|
|
309
|
+
|
|
310
|
+
name
|
|
311
|
+
end
|
|
312
|
+
|
|
313
|
+
def group_inputs_for_scope(group)
|
|
314
|
+
inputs = group.is_a?(Hash) ? group["inputs"] || group[:inputs] : nil
|
|
315
|
+
invalid!("coverage report combination group is malformed") unless inputs.is_a?(Array) && inputs.all?(String)
|
|
316
|
+
|
|
317
|
+
inputs
|
|
318
|
+
end
|
|
319
|
+
|
|
320
|
+
def projection_signature(rows)
|
|
321
|
+
signatures = rows.map do |row|
|
|
322
|
+
[row["id"], row["target"], row["inputs"], row["status"], row["hits"]]
|
|
323
|
+
end
|
|
324
|
+
signatures.sort_by { |id,| id.to_s }
|
|
325
|
+
end
|
|
326
|
+
|
|
327
|
+
def typed_inputs(inputs)
|
|
328
|
+
inputs.to_h do |name, value|
|
|
329
|
+
typed = Report.value(value)
|
|
330
|
+
[name.to_s, typed.to_h { |key, item| [key.to_s, item] }]
|
|
331
|
+
end
|
|
332
|
+
end
|
|
333
|
+
end
|
|
334
|
+
end
|
|
@@ -0,0 +1,124 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module SpecProof
|
|
4
|
+
# Normalizes and compares supported input domains.
|
|
5
|
+
module Domains
|
|
6
|
+
module_function
|
|
7
|
+
|
|
8
|
+
STRING_INSTANCE_OF = Object.instance_method(:instance_of?)
|
|
9
|
+
STRING_ENCODING = String.instance_method(:encoding)
|
|
10
|
+
STRING_VALID_ENCODING = String.instance_method(:valid_encoding?)
|
|
11
|
+
|
|
12
|
+
def supported_string?(value)
|
|
13
|
+
return false unless STRING_INSTANCE_OF.bind_call(value, String)
|
|
14
|
+
|
|
15
|
+
encoding = STRING_ENCODING.bind_call(value)
|
|
16
|
+
[Encoding::UTF_8, Encoding::US_ASCII].include?(encoding) && STRING_VALID_ENCODING.bind_call(value)
|
|
17
|
+
rescue TypeError
|
|
18
|
+
false
|
|
19
|
+
end
|
|
20
|
+
|
|
21
|
+
def canonical_string(value)
|
|
22
|
+
raise ArgumentError, "unsupported string value" unless supported_string?(value)
|
|
23
|
+
|
|
24
|
+
copy = String.new(value)
|
|
25
|
+
(copy.encoding == Encoding::US_ASCII ? copy.encode(Encoding::UTF_8) : copy).freeze
|
|
26
|
+
end
|
|
27
|
+
|
|
28
|
+
def from_range(range, source: nil)
|
|
29
|
+
unless range.is_a?(Range) && range.begin.is_a?(Integer) && range.end.is_a?(Integer)
|
|
30
|
+
raise ConfigurationError.new("range endpoints must be bounded integers", code: "invalid_range", source: source)
|
|
31
|
+
end
|
|
32
|
+
|
|
33
|
+
upper = range.exclude_end? ? range.end - 1 : range.end
|
|
34
|
+
IntegerDomain.new(lower: range.begin, upper: upper)
|
|
35
|
+
end
|
|
36
|
+
|
|
37
|
+
def from_values(values, source: nil)
|
|
38
|
+
unless values.is_a?(Array)
|
|
39
|
+
raise ConfigurationError.new("values must be an array", code: "invalid_values", source: source)
|
|
40
|
+
end
|
|
41
|
+
|
|
42
|
+
normalized = values.map do |value|
|
|
43
|
+
case value
|
|
44
|
+
when NilClass, TrueClass, FalseClass, Integer then value
|
|
45
|
+
when Symbol then value.to_s.freeze.to_sym
|
|
46
|
+
when String
|
|
47
|
+
unless supported_string?(value)
|
|
48
|
+
raise ConfigurationError.new("finite string values must be valid UTF-8 or US-ASCII strings",
|
|
49
|
+
code: "unsupported_value", source: source)
|
|
50
|
+
end
|
|
51
|
+
|
|
52
|
+
canonical_string(value)
|
|
53
|
+
else
|
|
54
|
+
raise ConfigurationError.new("finite values must be integers, symbols, strings, booleans, or nil",
|
|
55
|
+
code: "unsupported_value", source: source)
|
|
56
|
+
end
|
|
57
|
+
end
|
|
58
|
+
normalized.uniq!
|
|
59
|
+
FiniteDomain.new(values: normalized.sort_by { |value| sort_key(value) })
|
|
60
|
+
end
|
|
61
|
+
|
|
62
|
+
def sort_key(value)
|
|
63
|
+
case value
|
|
64
|
+
when NilClass then [0]
|
|
65
|
+
when FalseClass then [1, 0]
|
|
66
|
+
when TrueClass then [1, 1]
|
|
67
|
+
when Integer then [2, value]
|
|
68
|
+
when Symbol then [3, value.to_s]
|
|
69
|
+
when String then [4, value]
|
|
70
|
+
end
|
|
71
|
+
end
|
|
72
|
+
|
|
73
|
+
def include?(domain, value)
|
|
74
|
+
case domain
|
|
75
|
+
when IntegerDomain then value.is_a?(Integer) && value.between?(domain.lower, domain.upper)
|
|
76
|
+
when FiniteDomain
|
|
77
|
+
return false unless primitive?(value)
|
|
78
|
+
|
|
79
|
+
value = canonical_string(value) if supported_string?(value)
|
|
80
|
+
domain.values.any? { |member| member.instance_of?(value.class) && member.eql?(value) }
|
|
81
|
+
else false
|
|
82
|
+
end
|
|
83
|
+
end
|
|
84
|
+
|
|
85
|
+
def primitive?(value)
|
|
86
|
+
case value
|
|
87
|
+
when NilClass, TrueClass, FalseClass, Integer, Symbol then true
|
|
88
|
+
when String then supported_string?(value)
|
|
89
|
+
else false
|
|
90
|
+
end
|
|
91
|
+
end
|
|
92
|
+
|
|
93
|
+
def empty?(domain)
|
|
94
|
+
domain.is_a?(IntegerDomain) ? domain.lower > domain.upper : domain.values.empty?
|
|
95
|
+
end
|
|
96
|
+
|
|
97
|
+
def same_kind?(left, right)
|
|
98
|
+
(left.is_a?(IntegerDomain) && right.is_a?(IntegerDomain)) ||
|
|
99
|
+
(left.is_a?(FiniteDomain) && right.is_a?(FiniteDomain))
|
|
100
|
+
end
|
|
101
|
+
|
|
102
|
+
def to_a(domain)
|
|
103
|
+
domain.is_a?(IntegerDomain) ? [domain.lower, domain.upper] : domain.values
|
|
104
|
+
end
|
|
105
|
+
end
|
|
106
|
+
|
|
107
|
+
# Represents an inclusive integer interval, including an empty interval.
|
|
108
|
+
class IntegerDomain
|
|
109
|
+
def self.from_range(range)
|
|
110
|
+
Domains.from_range(range)
|
|
111
|
+
end
|
|
112
|
+
|
|
113
|
+
def include?(value)
|
|
114
|
+
value.is_a?(Integer) && value.between?(lower, upper)
|
|
115
|
+
end
|
|
116
|
+
end
|
|
117
|
+
|
|
118
|
+
# Represents a finite set of supported scalar values.
|
|
119
|
+
class FiniteDomain
|
|
120
|
+
def include?(value)
|
|
121
|
+
Domains.include?(self, value)
|
|
122
|
+
end
|
|
123
|
+
end
|
|
124
|
+
end
|