ast-merge 7.1.1 → 7.1.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- checksums.yaml.gz.sig +0 -0
- data/README.md +2 -9
- data/lib/ast/merge/portable_benchmark_contract.rb +514 -0
- data/lib/ast/merge/provider_contract.rb +196 -0
- data/lib/ast/merge/provider_registry.rb +177 -0
- data/lib/ast/merge/provider_result.rb +60 -0
- data/lib/ast/merge/rspec/provider_conformance.rb +84 -0
- data/lib/ast/merge/rspec/shared_examples/provider_conformance.rb +110 -0
- data/lib/ast/merge/rspec/shared_examples.rb +3 -0
- data/lib/ast/merge/source_render/fragment.rb +119 -0
- data/lib/ast/merge/source_render/plan.rb +110 -0
- data/lib/ast/merge/source_render/renderer.rb +196 -0
- data/lib/ast/merge/source_render/result.rb +35 -0
- data/lib/ast/merge/source_render.rb +21 -0
- data/lib/ast/merge/version.rb +1 -1
- data/lib/ast/merge/yaml_document.rb +159 -0
- data/lib/ast/merge.rb +23 -1
- data/sig/ast/merge.rbs +165 -0
- data.tar.gz.sig +0 -0
- metadata +26 -14
- metadata.gz.sig +2 -2
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 29d6492ada466c41cde1a0fdf7274c2b3e61b8b9b30f71102b20455f8552f5cc
|
|
4
|
+
data.tar.gz: cbd12e377b264c18022f6626a64e36b342d000d06dd756489232e35fa79ea668
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: d2fa0995982b3197f581eb0b414457cb3c7ce8944c0cd05417b565f6d0f30d6488f7b820731820ef039ec2c9fd9f9175c79510ae8a8c59448060be6f2fc53a63
|
|
7
|
+
data.tar.gz: 32772e9dc13aa1e89ae3a2e6a44a908a43126c2408e47659fb3ff7ae78bafff48f80c7bc756f985513ef694649a8c696b1dd490bdd10b8d74ce64f90791a14d8
|
checksums.yaml.gz.sig
CHANGED
|
Binary file
|
data/README.md
CHANGED
|
@@ -366,7 +366,7 @@ The `Ast::Merge` module is organized into several namespaces, each with detailed
|
|
|
366
366
|
|
|
367
367
|
### Compatibility
|
|
368
368
|
|
|
369
|
-
Compatible with MRI Ruby 4.0.0+,
|
|
369
|
+
Compatible with MRI Ruby 4.0.0+, JRuby, and TruffleRuby.
|
|
370
370
|
CI workflows and Appraisals are generated for MRI Ruby 4.0.0+.
|
|
371
371
|
This test floor is configured by `ruby.test_minimum` in `.kettle-jem.yml` and
|
|
372
372
|
may be higher than the gem's runtime compatibility floor when legacy Rubies are
|
|
@@ -514,20 +514,13 @@ See [SECURITY.md][🔐security].
|
|
|
514
514
|
## 🤝 Contributing
|
|
515
515
|
|
|
516
516
|
If you need some ideas of where to help, you could work on adding more code coverage,
|
|
517
|
-
or if it is already
|
|
517
|
+
or if it is already 💯, check [issues][🤝gh-issues] or [PRs][🤝gh-pulls],
|
|
518
518
|
or use the gem and think about how it could be better.
|
|
519
519
|
|
|
520
520
|
We [![Keep A Changelog][📗keep-changelog-img]][📗keep-changelog] so if you make changes, remember to update it.
|
|
521
521
|
|
|
522
522
|
See [CONTRIBUTING.md][🤝contributing] for more detailed instructions.
|
|
523
523
|
|
|
524
|
-
### Code Coverage
|
|
525
|
-
|
|
526
|
-
<details markdown="1">
|
|
527
|
-
<summary>Coverage service badges</summary>
|
|
528
|
-
|
|
529
|
-
</details>
|
|
530
|
-
|
|
531
524
|
## 📌 Versioning
|
|
532
525
|
|
|
533
526
|
This library follows [![Semantic Versioning 2.0.0][📌semver-img]][📌semver] for its public API where practical.
|
|
@@ -0,0 +1,514 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require 'digest'
|
|
4
|
+
require 'json'
|
|
5
|
+
|
|
6
|
+
module Ast
|
|
7
|
+
module Merge
|
|
8
|
+
# Offline parser and conformance gate for the portable benchmark exchange contract.
|
|
9
|
+
module PortableBenchmarkContract
|
|
10
|
+
SCHEMA_MAJOR = 1
|
|
11
|
+
SCHEMA_PATTERN = %r{\Astructuredmerge\.benchmark(?:\.[a-z_]+)?/v(\d+)(?:\.\d+)?\z}
|
|
12
|
+
ENUM_KEYS = %w[
|
|
13
|
+
kinds operations partitions profiles outcomes oracle_classes equivalence_classes transformations
|
|
14
|
+
false_auto_merge_severities preservation_requirements unsupported_policies budget_exceed_actions
|
|
15
|
+
llm_hard_gate_forbidden_profiles quality_dimensions
|
|
16
|
+
].freeze
|
|
17
|
+
QUALITY_EXCLUDED_OUTCOMES = %w[unsupported excluded_ambiguous].freeze
|
|
18
|
+
ADAPTER_DIGEST_FIELDS = %w[source_sha artifact_sha256 version configuration_sha256].freeze
|
|
19
|
+
HISTORY_FIELDS = %w[slice corpus_id manifest case_id merge_commit].freeze
|
|
20
|
+
SHA256_PATTERN = /\A[0-9a-f]{64}\z/
|
|
21
|
+
|
|
22
|
+
class ValidationError < Ast::Merge::Error; end
|
|
23
|
+
|
|
24
|
+
# Parses, validates, and deterministically interprets a contract without executing its payloads.
|
|
25
|
+
# rubocop:disable Metrics/AbcSize, Metrics/ClassLength, Metrics/CyclomaticComplexity, Metrics/MethodLength, Metrics/PerceivedComplexity -- Contract conformance keeps each envelope's validation together.
|
|
26
|
+
class Consumer
|
|
27
|
+
attr_reader :document, :contract_digest
|
|
28
|
+
|
|
29
|
+
def self.parse(source)
|
|
30
|
+
source = String(source)
|
|
31
|
+
new(JSON.parse(source), contract_digest: Digest::SHA256.hexdigest(source)).tap(&:validate!)
|
|
32
|
+
rescue JSON::ParserError => e
|
|
33
|
+
raise ValidationError, "invalid benchmark contract JSON: #{e.message}"
|
|
34
|
+
end
|
|
35
|
+
|
|
36
|
+
def self.load_file(path)
|
|
37
|
+
parse(File.binread(path))
|
|
38
|
+
end
|
|
39
|
+
|
|
40
|
+
def initialize(document, contract_digest: nil)
|
|
41
|
+
@document = document
|
|
42
|
+
@contract_digest = contract_digest || Digest::SHA256.hexdigest(canonical_json(document))
|
|
43
|
+
end
|
|
44
|
+
|
|
45
|
+
def validate!
|
|
46
|
+
object!(@document, 'contract')
|
|
47
|
+
validate_schema_versions!
|
|
48
|
+
validate_enum_contract!
|
|
49
|
+
validate_cases!
|
|
50
|
+
validate_run!
|
|
51
|
+
validate_results!
|
|
52
|
+
validate_report!
|
|
53
|
+
self
|
|
54
|
+
end
|
|
55
|
+
|
|
56
|
+
def canonical_summary
|
|
57
|
+
validate!
|
|
58
|
+
cases = fetch_array(@document, 'cases', 'contract')
|
|
59
|
+
results = fetch_array(@document, 'case_results', 'contract')
|
|
60
|
+
adapters = adapter_records
|
|
61
|
+
excluded = results.select { |result| QUALITY_EXCLUDED_OUTCOMES.include?(result['outcome']) }
|
|
62
|
+
false_auto_merges = results.filter_map do |result|
|
|
63
|
+
next unless result['outcome'] == 'false_auto_merge' && result['score_eligible']
|
|
64
|
+
|
|
65
|
+
{ 'id' => result.fetch('id'), 'severity' => result.dig('dimensions', 'safety', 'severity') }
|
|
66
|
+
end
|
|
67
|
+
|
|
68
|
+
{
|
|
69
|
+
'schema' => @document.fetch('schema_version'),
|
|
70
|
+
'counts' => {
|
|
71
|
+
'adapters' => adapters.length,
|
|
72
|
+
'cases' => cases.length,
|
|
73
|
+
'results' => results.length,
|
|
74
|
+
'selected_cases' => @document.dig('run_manifest', 'selection', 'selected_case_ids').length,
|
|
75
|
+
'score_eligible_results' => results.count { |result| result['score_eligible'] },
|
|
76
|
+
'quality_denominator_excluded_results' => excluded.length,
|
|
77
|
+
'eligible_false_auto_merges' => false_auto_merges.length
|
|
78
|
+
},
|
|
79
|
+
'case_ids_by_operation' => grouped_case_ids(cases, 'operations', 'operation'),
|
|
80
|
+
'case_ids_by_partition' => grouped_case_ids(cases, 'partitions', 'partition'),
|
|
81
|
+
'score_eligible_result_ids' => results.filter_map do |result|
|
|
82
|
+
result['id'] if result['score_eligible']
|
|
83
|
+
end.sort,
|
|
84
|
+
'quality_denominator_excluded_result_ids' => excluded.map { |result| result.fetch('id') }.sort,
|
|
85
|
+
'false_auto_merges' => false_auto_merges.sort_by { |entry| entry.fetch('id') },
|
|
86
|
+
'safety_gate' => {
|
|
87
|
+
'status' => false_auto_merges.empty? ? 'pass' : 'fail',
|
|
88
|
+
'eligible_false_auto_merge_count' => false_auto_merges.length,
|
|
89
|
+
'non_compensable' => true
|
|
90
|
+
},
|
|
91
|
+
'selection_reason_categories' => @document.dig('run_manifest', 'selection', 'explanation').keys.sort,
|
|
92
|
+
'contract_digest' => @contract_digest
|
|
93
|
+
}
|
|
94
|
+
end
|
|
95
|
+
|
|
96
|
+
alias summary canonical_summary
|
|
97
|
+
|
|
98
|
+
private
|
|
99
|
+
|
|
100
|
+
def validate_schema_versions!
|
|
101
|
+
schema_values = [@document['schema_version']]
|
|
102
|
+
schema_values.concat(fetch_array(@document, 'cases', 'contract').map { |entry| entry['schema_version'] })
|
|
103
|
+
run = fetch_hash(@document, 'run_manifest', 'contract')
|
|
104
|
+
report = fetch_hash(@document, 'aggregate_report', 'contract')
|
|
105
|
+
schema_values.concat([run['schema_version'], report['schema_version']])
|
|
106
|
+
|
|
107
|
+
schema_values.each_with_index do |schema, index|
|
|
108
|
+
match = SCHEMA_PATTERN.match(schema.to_s)
|
|
109
|
+
error!("schema version #{index} is invalid: #{schema.inspect}") unless match
|
|
110
|
+
error!("unsupported benchmark schema major v#{match[1]}") unless match[1].to_i == SCHEMA_MAJOR
|
|
111
|
+
end
|
|
112
|
+
end
|
|
113
|
+
|
|
114
|
+
def validate_enum_contract!
|
|
115
|
+
contract = fetch_hash(@document, 'contract', 'contract')
|
|
116
|
+
ENUM_KEYS.each do |key|
|
|
117
|
+
values = fetch_array(contract, key, 'contract enums')
|
|
118
|
+
error!("#{key} contains duplicate values") unless values.uniq.length == values.length
|
|
119
|
+
error!("#{key} must contain only non-empty strings") unless values.all? do |value|
|
|
120
|
+
value.is_a?(String) && !value.empty?
|
|
121
|
+
end
|
|
122
|
+
end
|
|
123
|
+
error!('scalar scores must be prohibited') unless contract['scalar_score_allowed'] == false
|
|
124
|
+
error!('false auto-merges must be non-compensable') unless contract['false_auto_merge_compensable'] == false
|
|
125
|
+
end
|
|
126
|
+
|
|
127
|
+
def validate_cases!
|
|
128
|
+
cases = fetch_array(@document, 'cases', 'contract')
|
|
129
|
+
validate_unique_ids!(cases, 'case')
|
|
130
|
+
case_schema = fetch_hash(@document, 'case_schema', 'contract')
|
|
131
|
+
required = fetch_array(case_schema, 'required', 'case schema')
|
|
132
|
+
operation_fields = fetch_hash(case_schema, 'operation_required_fields', 'case schema')
|
|
133
|
+
inline_limit = case_schema.dig('input_policy', 'inline_max_bytes')
|
|
134
|
+
|
|
135
|
+
cases.each do |benchmark_case|
|
|
136
|
+
id = benchmark_case['id']
|
|
137
|
+
required_fields!(benchmark_case, required, "case #{id}")
|
|
138
|
+
membership!(benchmark_case['kind'], 'kinds', "case #{id} kind")
|
|
139
|
+
error!("case #{id} kind must be benchmark_case") unless benchmark_case['kind'] == 'benchmark_case'
|
|
140
|
+
operation = membership!(benchmark_case['operation'], 'operations', "case #{id} operation")
|
|
141
|
+
membership!(benchmark_case['partition'], 'partitions', "case #{id} partition")
|
|
142
|
+
membership!(benchmark_case.dig('oracle', 'class'), 'oracle_classes', "case #{id} oracle")
|
|
143
|
+
boolean!(benchmark_case.dig('oracle', 'score_eligible'), "case #{id} oracle score eligibility")
|
|
144
|
+
membership!(benchmark_case['false_auto_merge_severity'], 'false_auto_merge_severities',
|
|
145
|
+
"case #{id} severity")
|
|
146
|
+
validate_capabilities!(benchmark_case, id)
|
|
147
|
+
validate_equivalence!(benchmark_case, id)
|
|
148
|
+
validate_preservation!(benchmark_case, id)
|
|
149
|
+
required_fields!(benchmark_case, operation_fields.fetch(operation), "case #{id} #{operation}")
|
|
150
|
+
validate_case_inline_records!(benchmark_case, id, inline_limit)
|
|
151
|
+
validate_history_reference!(benchmark_case, id) if operation == 'history_replay'
|
|
152
|
+
validate_transformations!(benchmark_case, id) if operation == 'metamorphic'
|
|
153
|
+
end
|
|
154
|
+
|
|
155
|
+
case_ids = cases.map { |entry| entry.fetch('id') }
|
|
156
|
+
cases.select { |entry| entry['operation'] == 'metamorphic' }.each do |entry|
|
|
157
|
+
reference!(entry['parent_case_id'], case_ids, "case #{entry['id']} parent_case_id")
|
|
158
|
+
end
|
|
159
|
+
end
|
|
160
|
+
|
|
161
|
+
def validate_run!
|
|
162
|
+
run = fetch_hash(@document, 'run_manifest', 'contract')
|
|
163
|
+
schema = fetch_hash(@document, 'run_manifest_schema', 'contract')
|
|
164
|
+
required_fields!(run, fetch_array(schema, 'required', 'run schema'), 'run manifest')
|
|
165
|
+
error!('run manifest kind must be run_manifest') unless run['kind'] == 'run_manifest'
|
|
166
|
+
membership!(run['profile'], 'profiles', 'run profile')
|
|
167
|
+
membership!(run['unsupported_policy'], 'unsupported_policies', 'unsupported policy')
|
|
168
|
+
|
|
169
|
+
adapter_required = fetch_array(schema, 'adapter_identity_required', 'run schema')
|
|
170
|
+
adapter_records.each do |adapter|
|
|
171
|
+
required_fields!(adapter, ['id', *adapter_required], "adapter #{adapter['id']}")
|
|
172
|
+
end
|
|
173
|
+
validate_unique_ids!(adapter_records, 'adapter')
|
|
174
|
+
|
|
175
|
+
cases = fetch_array(@document, 'cases', 'contract')
|
|
176
|
+
case_ids = cases.map { |entry| entry.fetch('id') }
|
|
177
|
+
selection = fetch_hash(run, 'selection', 'run manifest')
|
|
178
|
+
selected = fetch_array(selection, 'selected_case_ids', 'run selection')
|
|
179
|
+
excluded = fetch_array(selection, 'excluded_case_ids', 'run selection')
|
|
180
|
+
error!('selected case IDs contain duplicates') unless selected.uniq.length == selected.length
|
|
181
|
+
error!('excluded case IDs contain duplicates') unless excluded.uniq.length == excluded.length
|
|
182
|
+
error!('selected and excluded case IDs overlap') unless (selected & excluded).empty?
|
|
183
|
+
(selected + excluded).each { |id| reference!(id, case_ids, 'run selection case_id') }
|
|
184
|
+
validate_selection_explanation!(selection, case_ids, schema)
|
|
185
|
+
validate_llm_gate_profile!(run, cases, selected)
|
|
186
|
+
end
|
|
187
|
+
|
|
188
|
+
def validate_results!
|
|
189
|
+
results = fetch_array(@document, 'case_results', 'contract')
|
|
190
|
+
validate_unique_ids!(results, 'result')
|
|
191
|
+
required = fetch_array(fetch_hash(@document, 'case_result_schema', 'contract'), 'required', 'result schema')
|
|
192
|
+
run = fetch_hash(@document, 'run_manifest', 'contract')
|
|
193
|
+
cases = fetch_array(@document, 'cases', 'contract').to_h { |entry| [entry.fetch('id'), entry] }
|
|
194
|
+
adapter_ids = adapter_records.map { |entry| entry.fetch('id') }
|
|
195
|
+
|
|
196
|
+
results.each do |result|
|
|
197
|
+
id = result['id']
|
|
198
|
+
required_fields!(result, required, "result #{id}")
|
|
199
|
+
reference!(result['run_id'], [run['id']], "result #{id} run_id")
|
|
200
|
+
reference!(result['case_id'], cases.keys, "result #{id} case_id")
|
|
201
|
+
reference!(result['adapter_id'], adapter_ids, "result #{id} adapter_id")
|
|
202
|
+
outcome = membership!(result['outcome'], 'outcomes', "result #{id} outcome")
|
|
203
|
+
boolean!(result['score_eligible'], "result #{id} score eligibility")
|
|
204
|
+
validate_result_eligibility!(result, cases.fetch(result['case_id']), run)
|
|
205
|
+
validate_raw_inline_records!(result, id)
|
|
206
|
+
validate_false_auto_merge!(result, outcome)
|
|
207
|
+
end
|
|
208
|
+
end
|
|
209
|
+
|
|
210
|
+
def validate_report!
|
|
211
|
+
report = fetch_hash(@document, 'aggregate_report', 'contract')
|
|
212
|
+
schema = fetch_hash(@document, 'aggregate_report_schema', 'contract')
|
|
213
|
+
run = fetch_hash(@document, 'run_manifest', 'contract')
|
|
214
|
+
results = fetch_array(@document, 'case_results', 'contract')
|
|
215
|
+
result_ids = results.map { |entry| entry.fetch('id') }
|
|
216
|
+
error!('aggregate report kind must be aggregate_report') unless report['kind'] == 'aggregate_report'
|
|
217
|
+
reference!(report['run_id'], [run['id']], 'aggregate report run_id')
|
|
218
|
+
report_result_ids = fetch_array(report, 'raw_result_ids', 'aggregate report')
|
|
219
|
+
error!('aggregate raw result IDs must exactly cover results') unless report_result_ids.sort == result_ids.sort
|
|
220
|
+
unless report_result_ids.uniq.length == report_result_ids.length
|
|
221
|
+
error!('aggregate raw result IDs contain duplicates')
|
|
222
|
+
end
|
|
223
|
+
|
|
224
|
+
required_strata = fetch_array(schema, 'required_strata', 'report schema')
|
|
225
|
+
required_fields!(fetch_hash(report, 'stratification', 'aggregate report'), required_strata,
|
|
226
|
+
'report stratification')
|
|
227
|
+
validate_outcome_counts!(report, results)
|
|
228
|
+
validate_quality_denominators!(report, results, run)
|
|
229
|
+
validate_paired_deltas!(report, results, run)
|
|
230
|
+
validate_scalar_score!(report, schema)
|
|
231
|
+
validate_safety_gate!(report, results)
|
|
232
|
+
end
|
|
233
|
+
|
|
234
|
+
def validate_capabilities!(benchmark_case, id)
|
|
235
|
+
capabilities = fetch_array(benchmark_case, 'capabilities', "case #{id}")
|
|
236
|
+
error!("case #{id} capabilities must be unique and sorted") unless capabilities == capabilities.uniq.sort
|
|
237
|
+
end
|
|
238
|
+
|
|
239
|
+
def validate_equivalence!(benchmark_case, id)
|
|
240
|
+
fetch_array(benchmark_case, 'acceptable_equivalence', "case #{id}").each do |entry|
|
|
241
|
+
membership!(entry['class'], 'equivalence_classes', "case #{id} equivalence")
|
|
242
|
+
end
|
|
243
|
+
end
|
|
244
|
+
|
|
245
|
+
def validate_preservation!(benchmark_case, id)
|
|
246
|
+
preservation = fetch_hash(benchmark_case, 'preservation_policy', "case #{id}")
|
|
247
|
+
preservation.each_value do |requirement|
|
|
248
|
+
membership!(requirement, 'preservation_requirements', "case #{id} preservation")
|
|
249
|
+
end
|
|
250
|
+
end
|
|
251
|
+
|
|
252
|
+
def validate_transformations!(benchmark_case, id)
|
|
253
|
+
fetch_array(benchmark_case, 'transformations', "case #{id}").each do |transformation|
|
|
254
|
+
membership!(transformation['type'], 'transformations', "case #{id} transformation")
|
|
255
|
+
end
|
|
256
|
+
end
|
|
257
|
+
|
|
258
|
+
def validate_case_inline_records!(benchmark_case, id, inline_limit)
|
|
259
|
+
walk_hashes(benchmark_case) do |record|
|
|
260
|
+
next unless record['mode'] == 'inline'
|
|
261
|
+
|
|
262
|
+
bytes = record['bytes']
|
|
263
|
+
error!("case #{id} inline bytes must be a string") unless bytes.is_a?(String)
|
|
264
|
+
error!("case #{id} inline bytes exceed #{inline_limit}") if bytes.bytesize > inline_limit
|
|
265
|
+
digest!(record['sha256'], bytes, "case #{id} inline input")
|
|
266
|
+
end
|
|
267
|
+
end
|
|
268
|
+
|
|
269
|
+
def validate_raw_inline_records!(result, id)
|
|
270
|
+
raw = fetch_hash(result, 'raw', "result #{id}")
|
|
271
|
+
raw.each do |name, record|
|
|
272
|
+
next unless record.is_a?(Hash) && record.key?('inline')
|
|
273
|
+
|
|
274
|
+
inline = record['inline']
|
|
275
|
+
error!("result #{id} raw #{name} inline content must be a string") unless inline.is_a?(String)
|
|
276
|
+
error!("result #{id} raw #{name} byte length is not exact") unless record['bytes'] == inline.bytesize
|
|
277
|
+
digest!(record['sha256'], inline, "result #{id} raw #{name}")
|
|
278
|
+
end
|
|
279
|
+
end
|
|
280
|
+
|
|
281
|
+
def validate_history_reference!(benchmark_case, id)
|
|
282
|
+
history = fetch_hash(benchmark_case, 'history_reference', "case #{id}")
|
|
283
|
+
required_fields!(history, HISTORY_FIELDS, "case #{id} history reference")
|
|
284
|
+
error!("case #{id} history reference must target Slice 1021") unless history['slice'] == 1021
|
|
285
|
+
expected_manifest = 'diagnostics/slice-1021-reviewed-git-history-corpus/manifest.json'
|
|
286
|
+
error!("case #{id} history manifest must target Slice 1021") unless history['manifest'] == expected_manifest
|
|
287
|
+
error!("case #{id} history reference must not inline source bytes") if benchmark_case.key?('inputs')
|
|
288
|
+
end
|
|
289
|
+
|
|
290
|
+
def validate_selection_explanation!(selection, case_ids, schema)
|
|
291
|
+
explanation = fetch_hash(selection, 'explanation', 'run selection')
|
|
292
|
+
required_fields!(
|
|
293
|
+
explanation,
|
|
294
|
+
fetch_array(schema, 'fast_selection_required', 'run schema'),
|
|
295
|
+
'selection explanation'
|
|
296
|
+
)
|
|
297
|
+
explanation.fetch('direct_cases').each do |reason|
|
|
298
|
+
reason.fetch('case_ids').each { |id| reference!(id, case_ids, 'direct selection case_id') }
|
|
299
|
+
end
|
|
300
|
+
explanation.fetch('sentinels').each do |reason|
|
|
301
|
+
reference!(reason['case_id'], case_ids, 'sentinel selection case_id')
|
|
302
|
+
end
|
|
303
|
+
explanation.fetch('neighbor_samples').each do |sample|
|
|
304
|
+
(sample.fetch('population') + sample.fetch('selected_case_ids')).each do |id|
|
|
305
|
+
reference!(id, case_ids, 'neighbor selection case_id')
|
|
306
|
+
end
|
|
307
|
+
end
|
|
308
|
+
budget = fetch_hash(explanation, 'budget_exceed', 'selection explanation')
|
|
309
|
+
membership!(budget['action'], 'budget_exceed_actions', 'budget exceed action')
|
|
310
|
+
error!('budget exceed must regenerate the explanation') unless budget['regenerate_explanation'] == true
|
|
311
|
+
error!('silent budget extension is prohibited') unless budget['silent_extension'] == false
|
|
312
|
+
end
|
|
313
|
+
|
|
314
|
+
def validate_llm_gate_profile!(run, cases, selected)
|
|
315
|
+
forbidden = enum('llm_hard_gate_forbidden_profiles')
|
|
316
|
+
return unless forbidden.include?(run['profile'])
|
|
317
|
+
|
|
318
|
+
selected_cases = cases.select { |entry| selected.include?(entry['id']) }
|
|
319
|
+
llm_hard_gate = selected_cases.any? do |entry|
|
|
320
|
+
entry.dig('oracle', 'class') == 'llm' && entry.dig('oracle', 'score_eligible') == true
|
|
321
|
+
end
|
|
322
|
+
error!("LLM oracle cannot participate in #{run['profile']} hard gates") if llm_hard_gate
|
|
323
|
+
end
|
|
324
|
+
|
|
325
|
+
def validate_result_eligibility!(result, benchmark_case, run)
|
|
326
|
+
outcome = result['outcome']
|
|
327
|
+
expected = benchmark_case.dig('oracle', 'score_eligible') == true
|
|
328
|
+
expected = false if QUALITY_EXCLUDED_OUTCOMES.include?(outcome)
|
|
329
|
+
unless result['score_eligible'] == expected
|
|
330
|
+
error!("result #{result['id']} score eligibility conflicts with case/outcome")
|
|
331
|
+
end
|
|
332
|
+
if outcome == 'unsupported' && run['unsupported_policy'] == 'coverage_only' && result['score_eligible']
|
|
333
|
+
error!("result #{result['id']} unsupported outcome entered quality denominator")
|
|
334
|
+
end
|
|
335
|
+
end
|
|
336
|
+
|
|
337
|
+
def validate_false_auto_merge!(result, outcome)
|
|
338
|
+
safety = fetch_hash(fetch_hash(result, 'dimensions', "result #{result['id']}"), 'safety',
|
|
339
|
+
"result #{result['id']}")
|
|
340
|
+
if outcome == 'false_auto_merge'
|
|
341
|
+
error!("result #{result['id']} must identify a false auto-merge") unless safety['false_auto_merge'] == true
|
|
342
|
+
unless safety['compensable'] == false
|
|
343
|
+
error!("result #{result['id']} false auto-merge cannot be compensable")
|
|
344
|
+
end
|
|
345
|
+
membership!(safety['severity'], 'false_auto_merge_severities', "result #{result['id']} severity")
|
|
346
|
+
elsif safety['false_auto_merge'] == true
|
|
347
|
+
error!("result #{result['id']} safety classification conflicts with outcome")
|
|
348
|
+
end
|
|
349
|
+
end
|
|
350
|
+
|
|
351
|
+
def validate_outcome_counts!(report, results)
|
|
352
|
+
actual = enum('outcomes').to_h { |outcome| [outcome, results.count { |entry| entry['outcome'] == outcome }] }
|
|
353
|
+
error!('aggregate outcome counts do not match raw results') unless report['outcome_counts'] == actual
|
|
354
|
+
end
|
|
355
|
+
|
|
356
|
+
def validate_quality_denominators!(report, results, run)
|
|
357
|
+
eligible_count = results.count { |entry| entry['score_eligible'] }
|
|
358
|
+
%w[effectiveness safety preservation].each do |dimension|
|
|
359
|
+
actual = report.dig('dimensions', dimension, 'eligible')
|
|
360
|
+
error!("aggregate #{dimension} denominator is incorrect") unless actual == eligible_count
|
|
361
|
+
end
|
|
362
|
+
excluded = results.select { |entry| QUALITY_EXCLUDED_OUTCOMES.include?(entry['outcome']) }
|
|
363
|
+
error!('unsupported/excluded result entered quality denominator') if excluded.any? do |entry|
|
|
364
|
+
entry['score_eligible']
|
|
365
|
+
end
|
|
366
|
+
return unless run['unsupported_policy'] == 'coverage_only'
|
|
367
|
+
|
|
368
|
+
quality_failure = report.dig('dimensions', 'coverage', 'unsupported_is_quality_failure')
|
|
369
|
+
error!('coverage-only unsupported results cannot be quality failures') unless quality_failure == false
|
|
370
|
+
end
|
|
371
|
+
|
|
372
|
+
def validate_paired_deltas!(report, results, run)
|
|
373
|
+
by_id = results.to_h { |entry| [entry.fetch('id'), entry] }
|
|
374
|
+
pairs = fetch_array(report, 'paired_deltas', 'aggregate report')
|
|
375
|
+
pairs.each do |pair|
|
|
376
|
+
base = by_id[pair['base_result_id']]
|
|
377
|
+
candidate = by_id[pair['candidate_result_id']]
|
|
378
|
+
error!("paired delta #{pair['case_id']} has dangling base result") unless base
|
|
379
|
+
error!("paired delta #{pair['case_id']} has dangling candidate result") unless candidate
|
|
380
|
+
paired_case_ids = [base['case_id'], candidate['case_id']].uniq
|
|
381
|
+
error!("paired delta #{pair['case_id']} mixes cases") unless paired_case_ids == [pair['case_id']]
|
|
382
|
+
unless base['adapter_id'] == run.dig('base_adapter', 'id')
|
|
383
|
+
error!("paired delta #{pair['case_id']} base adapter is incorrect")
|
|
384
|
+
end
|
|
385
|
+
unless candidate['adapter_id'] == run.dig('candidate_adapter', 'id')
|
|
386
|
+
error!("paired delta #{pair['case_id']} candidate adapter is incorrect")
|
|
387
|
+
end
|
|
388
|
+
end
|
|
389
|
+
end
|
|
390
|
+
|
|
391
|
+
def validate_scalar_score!(report, schema)
|
|
392
|
+
error!('report schema must prohibit scalar scores') unless schema['scalar_score_allowed'] == false
|
|
393
|
+
error!('aggregate scalar score is prohibited') unless report['scalar_score'].nil?
|
|
394
|
+
end
|
|
395
|
+
|
|
396
|
+
def validate_safety_gate!(report, results)
|
|
397
|
+
false_auto_merges = results.count do |entry|
|
|
398
|
+
entry['score_eligible'] && entry['outcome'] == 'false_auto_merge'
|
|
399
|
+
end
|
|
400
|
+
gate = fetch_hash(fetch_hash(report, 'gates', 'aggregate report'), 'safety', 'aggregate gates')
|
|
401
|
+
expected_status = false_auto_merges.zero? ? 'pass' : 'fail'
|
|
402
|
+
unless gate['status'] == expected_status
|
|
403
|
+
error!('safety gate status does not reflect eligible false auto-merges')
|
|
404
|
+
end
|
|
405
|
+
unless gate['eligible_false_auto_merge_count'] == false_auto_merges
|
|
406
|
+
error!('safety gate false-auto-merge count is incorrect')
|
|
407
|
+
end
|
|
408
|
+
error!('safety gate must be non-compensable') unless gate['non_compensable'] == true
|
|
409
|
+
error!('safety gate must prohibit offsets') unless gate['offsets_allowed'] == []
|
|
410
|
+
end
|
|
411
|
+
|
|
412
|
+
def grouped_case_ids(cases, enum_key, field)
|
|
413
|
+
enum(enum_key).to_h do |value|
|
|
414
|
+
[value, cases.filter_map { |entry| entry['id'] if entry[field] == value }.sort]
|
|
415
|
+
end
|
|
416
|
+
end
|
|
417
|
+
|
|
418
|
+
def adapter_records
|
|
419
|
+
run = fetch_hash(@document, 'run_manifest', 'contract')
|
|
420
|
+
[run['candidate_adapter'], run['base_adapter']].compact
|
|
421
|
+
end
|
|
422
|
+
|
|
423
|
+
def enum(key)
|
|
424
|
+
fetch_array(fetch_hash(@document, 'contract', 'contract'), key, 'contract enums')
|
|
425
|
+
end
|
|
426
|
+
|
|
427
|
+
def membership!(value, enum_key, label)
|
|
428
|
+
unless enum(enum_key).include?(value)
|
|
429
|
+
error!("#{label} is not a declared #{enum_key} member: #{value.inspect}")
|
|
430
|
+
end
|
|
431
|
+
value
|
|
432
|
+
end
|
|
433
|
+
|
|
434
|
+
def required_fields!(object, fields, label)
|
|
435
|
+
fields.each do |field|
|
|
436
|
+
value = field.to_s.split('.').reduce(object) { |memo, part| memo.is_a?(Hash) ? memo[part] : nil }
|
|
437
|
+
error!("#{label} is missing required field #{field}") if value.nil?
|
|
438
|
+
end
|
|
439
|
+
end
|
|
440
|
+
|
|
441
|
+
def validate_unique_ids!(records, label)
|
|
442
|
+
ids = records.map { |entry| entry['id'] }
|
|
443
|
+
error!("#{label} IDs must be non-empty strings") unless ids.all? { |id| id.is_a?(String) && !id.empty? }
|
|
444
|
+
error!("duplicate #{label} ID") unless ids.uniq.length == ids.length
|
|
445
|
+
end
|
|
446
|
+
|
|
447
|
+
def reference!(value, allowed, label)
|
|
448
|
+
error!("#{label} is dangling: #{value.inspect}") unless allowed.include?(value)
|
|
449
|
+
end
|
|
450
|
+
|
|
451
|
+
def digest!(digest, content, label)
|
|
452
|
+
error!("#{label} SHA-256 is malformed") unless SHA256_PATTERN.match?(digest.to_s)
|
|
453
|
+
error!("#{label} SHA-256 does not match exact bytes") unless digest == Digest::SHA256.hexdigest(content)
|
|
454
|
+
end
|
|
455
|
+
|
|
456
|
+
def boolean!(value, label)
|
|
457
|
+
error!("#{label} must be boolean") unless [true, false].include?(value)
|
|
458
|
+
end
|
|
459
|
+
|
|
460
|
+
def fetch_hash(object, key, label)
|
|
461
|
+
value = object[key]
|
|
462
|
+
error!("#{label}.#{key} must be an object") unless value.is_a?(Hash)
|
|
463
|
+
value
|
|
464
|
+
end
|
|
465
|
+
|
|
466
|
+
def fetch_array(object, key, label)
|
|
467
|
+
value = object[key]
|
|
468
|
+
error!("#{label}.#{key} must be an array") unless value.is_a?(Array)
|
|
469
|
+
value
|
|
470
|
+
end
|
|
471
|
+
|
|
472
|
+
def object!(value, label)
|
|
473
|
+
error!("#{label} must be an object") unless value.is_a?(Hash)
|
|
474
|
+
end
|
|
475
|
+
|
|
476
|
+
def walk_hashes(value, &block)
|
|
477
|
+
case value
|
|
478
|
+
when Hash
|
|
479
|
+
yield value
|
|
480
|
+
value.each_value { |child| walk_hashes(child, &block) }
|
|
481
|
+
when Array
|
|
482
|
+
value.each { |child| walk_hashes(child, &block) }
|
|
483
|
+
end
|
|
484
|
+
end
|
|
485
|
+
|
|
486
|
+
def canonical_json(value)
|
|
487
|
+
case value
|
|
488
|
+
when Hash
|
|
489
|
+
JSON.generate(value.keys.sort.to_h { |key| [key, JSON.parse(canonical_json(value.fetch(key)))] })
|
|
490
|
+
when Array
|
|
491
|
+
JSON.generate(value.map { |entry| JSON.parse(canonical_json(entry)) })
|
|
492
|
+
else
|
|
493
|
+
JSON.generate(value)
|
|
494
|
+
end
|
|
495
|
+
end
|
|
496
|
+
|
|
497
|
+
def error!(message)
|
|
498
|
+
raise ValidationError, message
|
|
499
|
+
end
|
|
500
|
+
end
|
|
501
|
+
# rubocop:enable Metrics/AbcSize, Metrics/ClassLength, Metrics/CyclomaticComplexity, Metrics/MethodLength, Metrics/PerceivedComplexity
|
|
502
|
+
|
|
503
|
+
module_function
|
|
504
|
+
|
|
505
|
+
def parse(source)
|
|
506
|
+
Consumer.parse(source)
|
|
507
|
+
end
|
|
508
|
+
|
|
509
|
+
def load_file(path)
|
|
510
|
+
Consumer.load_file(path)
|
|
511
|
+
end
|
|
512
|
+
end
|
|
513
|
+
end
|
|
514
|
+
end
|