featureparity 0.0.16 → 0.0.17
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/fp/cli.rb +2 -0
- data/lib/fp/client.rb +43 -3
- data/lib/fp/commands/changed_tests.rb +126 -0
- data/lib/fp/commands/ci_report.rb +133 -28
- data/lib/fp/commands/comment.rb +99 -0
- data/lib/fp/commands/help.rb +33 -0
- data/lib/fp/commands/init.rb +52 -12
- data/lib/fp/commands/propose_change.rb +46 -1
- data/lib/fp/commands/report.rb +63 -6
- data/lib/fp/commands.rb +2 -0
- data/lib/fp/junit.rb +50 -5
- data/lib/fp/source_fingerprint.rb +193 -0
- data/lib/fp/version.rb +1 -1
- data/lib/fp.rb +1 -0
- data/skills/fp/SKILL.md +31 -0
- data/skills/fp/references/cli.md +61 -1
- metadata +5 -2
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: d20b326add84bbad1fc34a9b2b41c70216f7e9f2797f47cc9cba9f4293d27f1e
|
|
4
|
+
data.tar.gz: ae9721b86841174ee64f1925b0878e5ca5b5e34ea1f467ea0275887526515695
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 87aa7b100433761979acec4c6a3f66c665757ebd77924cf1bab483861a1855be92acba93d3a6bd0e2f0cdc9daf9c887380a808eab96c467c3a41ee4e665f8bb0
|
|
7
|
+
data.tar.gz: 640c69700985e08ce0c6aec534d42d544db0d04771ae9f9be254e24aca3952cbbff7f52456c1c7b506c8bbf5e6a3d3b54c8c7959d6575d159d19cff5cad69c0b
|
data/lib/fp/cli.rb
CHANGED
|
@@ -11,10 +11,12 @@ module Fp
|
|
|
11
11
|
'show' => Commands::Show,
|
|
12
12
|
'propose' => Commands::Propose,
|
|
13
13
|
'propose-change' => Commands::ProposeChange,
|
|
14
|
+
'comment' => Commands::Comment,
|
|
14
15
|
'history' => Commands::History,
|
|
15
16
|
'changes' => Commands::Changes,
|
|
16
17
|
'report' => Commands::Report,
|
|
17
18
|
'ci-report' => Commands::CiReport,
|
|
19
|
+
'changed-tests' => Commands::ChangedTests,
|
|
18
20
|
'check' => Commands::Check,
|
|
19
21
|
'matrix' => Commands::Matrix,
|
|
20
22
|
'export' => Commands::Export,
|
data/lib/fp/client.rb
CHANGED
|
@@ -130,6 +130,15 @@ module Fp
|
|
|
130
130
|
put("/api/requirements/#{requirement_id}/changes/#{change_id}", params)
|
|
131
131
|
end
|
|
132
132
|
|
|
133
|
+
# POST /requirements/:id/changes/:change_id/comment
|
|
134
|
+
#
|
|
135
|
+
# Thread a comment onto the open proposal. The single-open-proposal rule means a
|
|
136
|
+
# second agent can't open a rival change; this lets it raise its concern *on* the
|
|
137
|
+
# proposal a human is about to review instead of hitting a dead-end 409.
|
|
138
|
+
def comment_requirement_change(requirement_id, change_id, params)
|
|
139
|
+
post("/api/requirements/#{requirement_id}/changes/#{change_id}/comment", params)
|
|
140
|
+
end
|
|
141
|
+
|
|
133
142
|
# GET /projects/:project_id/evidence — the receipts already filed for a project,
|
|
134
143
|
# optionally narrowed to one requirement slug and/or surface.
|
|
135
144
|
def list_evidence(project_id, slug: nil, surface: nil)
|
|
@@ -160,7 +169,7 @@ module Fp
|
|
|
160
169
|
# only do that by fetching the test files from GitHub, which fails silently on a
|
|
161
170
|
# private repo, so resolved results are always sent when we have them.
|
|
162
171
|
def ingest_ci_results(project_id, ci_url:, surface:, junit_xml:, pr_url: nil, results: nil,
|
|
163
|
-
unmarked_failures: nil)
|
|
172
|
+
test_revisions: nil, unmarked_failures: nil)
|
|
164
173
|
body = {
|
|
165
174
|
ci_url: ci_url,
|
|
166
175
|
surface: surface,
|
|
@@ -169,6 +178,9 @@ module Fp
|
|
|
169
178
|
# Only send pr_url when set, so a merged-branch run stays a plain ingest.
|
|
170
179
|
body[:pr_url] = pr_url if pr_url && !pr_url.empty?
|
|
171
180
|
body[:results] = results if results && !results.empty?
|
|
181
|
+
# Testcase-level source receipts stay separate from aggregate Evidence outcomes:
|
|
182
|
+
# several tests may back one cell, and each can change independently.
|
|
183
|
+
body[:test_revisions] = test_revisions if test_revisions && !test_revisions.empty?
|
|
172
184
|
# Loose ends: failing tests with no fp:<slug> marker. The server stores them so the
|
|
173
185
|
# matrix can surface a red suite whose failures aren't bound to any requirement.
|
|
174
186
|
body[:unmarked_failures] = unmarked_failures if unmarked_failures && !unmarked_failures.empty?
|
|
@@ -176,6 +188,27 @@ module Fp
|
|
|
176
188
|
post("/api/projects/#{project_id}/ci_results", body)
|
|
177
189
|
end
|
|
178
190
|
|
|
191
|
+
# GET /projects/:project_id/changed_test_findings — every unresolved audit finding.
|
|
192
|
+
def list_changed_test_findings(project_id, surface: nil, test_type: nil)
|
|
193
|
+
params = {}
|
|
194
|
+
params[:surface] = surface if surface
|
|
195
|
+
params[:test_type] = test_type if test_type
|
|
196
|
+
get("/api/projects/#{project_id}/changed_test_findings", params)
|
|
197
|
+
end
|
|
198
|
+
|
|
199
|
+
# POST an assessment for the exact candidate fingerprint the agent inspected.
|
|
200
|
+
def assess_changed_test_finding(project_id, finding_id, classification:, explanation:,
|
|
201
|
+
candidate_fingerprint:)
|
|
202
|
+
post(
|
|
203
|
+
"/api/projects/#{project_id}/changed_test_findings/#{finding_id}/assessment",
|
|
204
|
+
{
|
|
205
|
+
classification: classification,
|
|
206
|
+
explanation: explanation,
|
|
207
|
+
candidate_fingerprint: candidate_fingerprint
|
|
208
|
+
}
|
|
209
|
+
)
|
|
210
|
+
end
|
|
211
|
+
|
|
179
212
|
# GET /projects/:project_id/parity — the computed parity grid and gate verdict.
|
|
180
213
|
#
|
|
181
214
|
# This replaces the old get_matrix/get_gaps stubs, which pointed at /api/matrix
|
|
@@ -205,7 +238,10 @@ module Fp
|
|
|
205
238
|
# ingest_ci_results). Sending them is what lets a surface whose repo isn't registered
|
|
206
239
|
# — or a private one the server can't read — be previewed at all.
|
|
207
240
|
def preview_parity(project_id, surface:, junit_xmls:, level: nil, results: nil)
|
|
208
|
-
|
|
241
|
+
# Older API deployments read `surface_filter` for report scoping while `surface`
|
|
242
|
+
# identifies the run's overlay. Send both during the rollout; current servers use
|
|
243
|
+
# `surface` for both meanings.
|
|
244
|
+
body = { surface: surface, surface_filter: surface, junit_xmls: Array(junit_xmls) }
|
|
209
245
|
body[:results] = results if results && !results.empty?
|
|
210
246
|
body[:level] = level if level
|
|
211
247
|
post("/api/projects/#{project_id}/parity/preview", body)
|
|
@@ -360,7 +396,11 @@ module Fp
|
|
|
360
396
|
else
|
|
361
397
|
body.to_s.empty? ? "HTTP #{status}" : body.to_s[0..200]
|
|
362
398
|
end
|
|
363
|
-
{ ok: false, error: error_message, status: status }
|
|
399
|
+
result = { ok: false, error: error_message, status: status }
|
|
400
|
+
# Carry structured error details (e.g. the open proposal behind a 409) so a
|
|
401
|
+
# command can act on them instead of only printing the message.
|
|
402
|
+
result[:details] = data['details'] if data.is_a?(Hash) && data.key?('details')
|
|
403
|
+
result
|
|
364
404
|
end
|
|
365
405
|
end
|
|
366
406
|
end
|
|
@@ -0,0 +1,126 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Fp
|
|
4
|
+
module Commands
|
|
5
|
+
# Lists CI-detected revisions to bound tests and records an agent's assessment of an
|
|
6
|
+
# exact candidate fingerprint.
|
|
7
|
+
class ChangedTests < Base
|
|
8
|
+
CLASSIFICATIONS = %w[still_proves no_longer_proves uncertain].freeze
|
|
9
|
+
KNOWN_FLAGS = {
|
|
10
|
+
project: :string,
|
|
11
|
+
surface: :string,
|
|
12
|
+
test_type: :string,
|
|
13
|
+
classification: :string,
|
|
14
|
+
explanation: :string,
|
|
15
|
+
candidate_fingerprint: :string
|
|
16
|
+
}.freeze
|
|
17
|
+
|
|
18
|
+
def run(args)
|
|
19
|
+
opts, positional = parse_flags(args, KNOWN_FLAGS)
|
|
20
|
+
require_project!(opts)
|
|
21
|
+
project_id = resolve_project_id(opts[:project])
|
|
22
|
+
|
|
23
|
+
if positional.first == 'assess'
|
|
24
|
+
assess(project_id, positional[1], opts)
|
|
25
|
+
else
|
|
26
|
+
list(project_id, opts)
|
|
27
|
+
end
|
|
28
|
+
end
|
|
29
|
+
|
|
30
|
+
private
|
|
31
|
+
|
|
32
|
+
def list(project_id, opts)
|
|
33
|
+
result = client.list_changed_test_findings(
|
|
34
|
+
project_id,
|
|
35
|
+
surface: opts[:surface],
|
|
36
|
+
test_type: opts[:test_type]
|
|
37
|
+
)
|
|
38
|
+
fail_result(result) unless result[:ok]
|
|
39
|
+
|
|
40
|
+
data = result[:data] || {}
|
|
41
|
+
findings = data['changed_test_findings'] || []
|
|
42
|
+
output.success(data) do
|
|
43
|
+
if findings.empty?
|
|
44
|
+
puts 'No unresolved changed-test findings.'
|
|
45
|
+
next
|
|
46
|
+
end
|
|
47
|
+
|
|
48
|
+
puts "#{findings.length} unresolved changed-test audit finding(s):"
|
|
49
|
+
findings.each { |finding| print_finding(finding) }
|
|
50
|
+
end
|
|
51
|
+
end
|
|
52
|
+
|
|
53
|
+
def assess(project_id, finding_id, opts)
|
|
54
|
+
if finding_id.nil? || finding_id.empty?
|
|
55
|
+
output.error('Finding id is required. Usage: fp changed-tests assess <id> ...')
|
|
56
|
+
exit 1
|
|
57
|
+
end
|
|
58
|
+
|
|
59
|
+
classification = opts[:classification].to_s.tr('-', '_')
|
|
60
|
+
unless CLASSIFICATIONS.include?(classification)
|
|
61
|
+
output.error("--classification must be one of: #{CLASSIFICATIONS.join(', ')}")
|
|
62
|
+
exit 1
|
|
63
|
+
end
|
|
64
|
+
require_flag(opts, :candidate_fingerprint)
|
|
65
|
+
require_flag(opts, :explanation)
|
|
66
|
+
if opts[:explanation].to_s.strip.empty?
|
|
67
|
+
output.error('--explanation must not be blank')
|
|
68
|
+
exit 1
|
|
69
|
+
end
|
|
70
|
+
|
|
71
|
+
result = client.assess_changed_test_finding(
|
|
72
|
+
project_id,
|
|
73
|
+
finding_id,
|
|
74
|
+
classification: classification,
|
|
75
|
+
explanation: opts[:explanation],
|
|
76
|
+
candidate_fingerprint: opts[:candidate_fingerprint]
|
|
77
|
+
)
|
|
78
|
+
fail_result(result) unless result[:ok]
|
|
79
|
+
|
|
80
|
+
data = result[:data] || {}
|
|
81
|
+
output.success(data) do
|
|
82
|
+
finding = data['changed_test_finding'] || {}
|
|
83
|
+
puts "Recorded #{classification.tr('_', ' ')} for changed-test finding #{finding_id}."
|
|
84
|
+
puts "Status: #{finding['status']}"
|
|
85
|
+
puts 'The CI outcome and requirement wording were not changed.'
|
|
86
|
+
end
|
|
87
|
+
end
|
|
88
|
+
|
|
89
|
+
def print_finding(finding)
|
|
90
|
+
requirement = finding['requirement'] || {}
|
|
91
|
+
testcase = finding['testcase'] || {}
|
|
92
|
+
accepted = finding['accepted'] || {}
|
|
93
|
+
candidate = finding['candidate'] || {}
|
|
94
|
+
assessment = finding['assessment'] || {}
|
|
95
|
+
|
|
96
|
+
puts
|
|
97
|
+
puts "#{finding['id']} #{requirement['slug']} [#{finding['surface']}#{type_suffix(finding)}]"
|
|
98
|
+
puts " Requirement: #{requirement['title']}"
|
|
99
|
+
puts " Acceptance: #{requirement['acceptance']}"
|
|
100
|
+
puts " Testcase: #{testcase['name']} (#{testcase['outcome']})"
|
|
101
|
+
puts " Scope: #{finding['fingerprint_scope']}"
|
|
102
|
+
puts " Accepted: #{revision_text(accepted)}"
|
|
103
|
+
puts " Candidate: #{revision_text(candidate)}"
|
|
104
|
+
puts " Assessment: #{assessment['classification']}: #{assessment['explanation']}" unless assessment.empty?
|
|
105
|
+
puts ' Assess: fp changed-tests assess ' \
|
|
106
|
+
"#{finding['id']} --candidate-fingerprint #{candidate['fingerprint']} " \
|
|
107
|
+
'--classification still_proves|no_longer_proves|uncertain --explanation "..."'
|
|
108
|
+
end
|
|
109
|
+
|
|
110
|
+
def type_suffix(finding)
|
|
111
|
+
type = finding['test_type'].to_s
|
|
112
|
+
type.empty? || type == 'unspecified' ? '' : ":#{type}"
|
|
113
|
+
end
|
|
114
|
+
|
|
115
|
+
def revision_text(revision)
|
|
116
|
+
location = revision['source_url'] || [revision['source_file'], revision['start_line']].compact.join(':')
|
|
117
|
+
"#{revision['fingerprint']} @ #{revision['commit_sha']} #{location}".strip
|
|
118
|
+
end
|
|
119
|
+
|
|
120
|
+
def fail_result(result)
|
|
121
|
+
output.error(result[:error], status: result[:status])
|
|
122
|
+
exit 1
|
|
123
|
+
end
|
|
124
|
+
end
|
|
125
|
+
end
|
|
126
|
+
end
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
3
|
require 'set'
|
|
4
|
+
require 'open3'
|
|
4
5
|
|
|
5
6
|
module Fp
|
|
6
7
|
module Commands
|
|
@@ -34,8 +35,10 @@ module Fp
|
|
|
34
35
|
# `fp report`.
|
|
35
36
|
#
|
|
36
37
|
# Multiple --junit flags may be passed to ingest several reports for the same
|
|
37
|
-
# surface in one invocation (e.g.
|
|
38
|
-
# uploaded
|
|
38
|
+
# surface in one invocation (e.g. unit and e2e suites, or test shards). They are
|
|
39
|
+
# resolved and uploaded as one run. This is load-bearing: each API ingest performs
|
|
40
|
+
# the vanished-test sweep, so uploading partitions independently would make every
|
|
41
|
+
# partition call the passing tests in its siblings "missing".
|
|
39
42
|
#
|
|
40
43
|
# --base-dir tells the CLI where the report's relative test-file paths resolve from
|
|
41
44
|
# (vitest, for instance, reports paths relative to the frontend package, not the repo
|
|
@@ -55,6 +58,8 @@ module Fp
|
|
|
55
58
|
pr: :string,
|
|
56
59
|
junit: :string,
|
|
57
60
|
base_dir: :string,
|
|
61
|
+
repo: :string,
|
|
62
|
+
sha: :string,
|
|
58
63
|
allow_empty: :boolean
|
|
59
64
|
}.freeze
|
|
60
65
|
|
|
@@ -81,42 +86,64 @@ module Fp
|
|
|
81
86
|
|
|
82
87
|
project_id = resolve_project_id(opts[:project])
|
|
83
88
|
known = known_slugs(project_id)
|
|
89
|
+
commit_sha = resolve_commit_sha(opts)
|
|
90
|
+
repo = opts[:repo] || presence(ENV['GITHUB_REPOSITORY'])
|
|
84
91
|
|
|
85
92
|
totals = { created: 0, updated: 0, missing_failing: 0, reports: [], resolved: 0,
|
|
86
|
-
unmatched: 0, unknown: [], unmarked_failures: []
|
|
93
|
+
unmatched: 0, unknown: [], unmarked_failures: [], test_revisions: 0,
|
|
94
|
+
findings_opened: 0, baselines_accepted: 0 }
|
|
87
95
|
|
|
88
|
-
junit_paths.
|
|
89
|
-
|
|
90
|
-
resolved = resolve_results(path, opts)
|
|
96
|
+
resolved_reports = junit_paths.map do |path|
|
|
97
|
+
resolved = resolve_results(path, opts, commit_sha: commit_sha, repo: repo)
|
|
91
98
|
totals[:unmatched] += resolved[:unmatched]
|
|
92
99
|
|
|
93
100
|
results, unknown = partition_known(resolved[:results], known)
|
|
101
|
+
test_revisions, revision_unknown = partition_known(resolved[:test_revisions], known)
|
|
94
102
|
totals[:resolved] += results.length
|
|
95
|
-
totals[:unknown].concat(unknown)
|
|
103
|
+
totals[:unknown].concat(unknown).concat(revision_unknown)
|
|
96
104
|
totals[:unmarked_failures].concat(resolved[:unmarked_failures])
|
|
105
|
+
totals[:test_revisions] += test_revisions.length
|
|
97
106
|
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
107
|
+
{ path: path, xml: File.read(path), results: results, test_revisions: test_revisions }
|
|
108
|
+
end
|
|
109
|
+
|
|
110
|
+
# One surface run may be split across several reports (unit + e2e, shards, etc.).
|
|
111
|
+
# The API's vanished-test sweep must see their union in one request; posting each
|
|
112
|
+
# partition separately makes the tests in every sibling report appear to have
|
|
113
|
+
# disappeared. Duplicate slug/surface/type claims are collapsed conservatively.
|
|
114
|
+
combined_results = combine_results(resolved_reports.flat_map { |report| report[:results] })
|
|
115
|
+
combined_revisions = combine_test_revisions(
|
|
116
|
+
resolved_reports.flat_map { |report| report[:test_revisions] }
|
|
117
|
+
)
|
|
118
|
+
result = client.ingest_ci_results(
|
|
119
|
+
project_id,
|
|
120
|
+
ci_url: opts[:ci_url],
|
|
121
|
+
surface: opts[:surface],
|
|
122
|
+
# Resolved results are authoritative. Keep one valid raw report for compatibility
|
|
123
|
+
# with servers old enough to ignore client-resolved results.
|
|
124
|
+
junit_xml: resolved_reports.first[:xml],
|
|
125
|
+
pr_url: opts[:pr],
|
|
126
|
+
results: combined_results,
|
|
127
|
+
test_revisions: combined_revisions,
|
|
128
|
+
unmarked_failures: totals[:unmarked_failures]
|
|
129
|
+
)
|
|
112
130
|
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
totals[:missing_failing] += data['missing_marked_failing'].to_i
|
|
117
|
-
totals[:reports] << { path: path, results: data['results'] || [] }
|
|
131
|
+
unless result[:ok]
|
|
132
|
+
output.error("Failed to ingest #{junit_paths.join(', ')}: #{result[:error]}", status: result[:status])
|
|
133
|
+
exit 1
|
|
118
134
|
end
|
|
119
135
|
|
|
136
|
+
data = result[:data] || {}
|
|
137
|
+
# Belt's implicit instance-variable serializer wrapped this response in older
|
|
138
|
+
# deployments. Accept both shapes while the API now returns the result directly.
|
|
139
|
+
data = data['ingest_result'] if data['ingest_result'].is_a?(Hash)
|
|
140
|
+
totals[:created] += data['evidence_created'].to_i
|
|
141
|
+
totals[:updated] += data['evidence_updated'].to_i
|
|
142
|
+
totals[:missing_failing] += data['missing_marked_failing'].to_i
|
|
143
|
+
totals[:findings_opened] += data['changed_test_findings_opened'].to_i
|
|
144
|
+
totals[:baselines_accepted] += data['bound_test_baselines_accepted'].to_i
|
|
145
|
+
totals[:reports] << { paths: junit_paths, results: data['results'] || [] }
|
|
146
|
+
|
|
120
147
|
# Reporting nothing is the failure this command exists to make visible: a run that
|
|
121
148
|
# resolves no markers has told the matrix nothing, and used to do so silently.
|
|
122
149
|
if totals[:resolved].zero? && !opts[:allow_empty]
|
|
@@ -194,6 +221,29 @@ module Fp
|
|
|
194
221
|
(result[:data]['requirements'] || []).map { |r| r['slug'] }.compact.to_set
|
|
195
222
|
end
|
|
196
223
|
|
|
224
|
+
# Collapse the same marker key across report partitions before uploading it. A red
|
|
225
|
+
# claim wins; otherwise any executed passing test wins over an all-skipped stub.
|
|
226
|
+
# Counts are summed so the evidence still describes the complete surface run, and
|
|
227
|
+
# the source points at the claim that determined the verdict.
|
|
228
|
+
def combine_results(results)
|
|
229
|
+
severity = { 'failing' => 0, 'passing' => 1, 'stub' => 2 }
|
|
230
|
+
|
|
231
|
+
results.each_with_object({}) do |result, combined|
|
|
232
|
+
key = [result[:slug], result[:surface], result[:test_type]]
|
|
233
|
+
current = combined[key]
|
|
234
|
+
unless current
|
|
235
|
+
combined[key] = result.dup
|
|
236
|
+
next
|
|
237
|
+
end
|
|
238
|
+
|
|
239
|
+
decisive = [current, result].min_by { |entry| severity.fetch(entry[:status], 0) }
|
|
240
|
+
merged = decisive.dup
|
|
241
|
+
merged[:test_count] = current[:test_count].to_i + result[:test_count].to_i
|
|
242
|
+
merged[:failure_count] = current[:failure_count].to_i + result[:failure_count].to_i
|
|
243
|
+
combined[key] = merged
|
|
244
|
+
end.values
|
|
245
|
+
end
|
|
246
|
+
|
|
197
247
|
def partition_known(results, known)
|
|
198
248
|
return [results, []] if known.nil?
|
|
199
249
|
|
|
@@ -201,6 +251,15 @@ module Fp
|
|
|
201
251
|
[kept, unknown.map { |r| r[:slug] }.uniq]
|
|
202
252
|
end
|
|
203
253
|
|
|
254
|
+
# Reports can be split into shards. Keep every independently identified testcase,
|
|
255
|
+
# but collapse an exact duplicate emitted by overlapping reports.
|
|
256
|
+
def combine_test_revisions(revisions)
|
|
257
|
+
revisions.each_with_object({}) do |revision, combined|
|
|
258
|
+
key = [revision[:slug], revision[:surface], revision[:test_type], revision[:testcase_key]]
|
|
259
|
+
combined[key] = revision
|
|
260
|
+
end.values
|
|
261
|
+
end
|
|
262
|
+
|
|
204
263
|
# A warning that survives a CI log. Stderr always; a GitHub Actions annotation too
|
|
205
264
|
# when we're running in one, so it shows up in the run summary rather than scrolling
|
|
206
265
|
# past in a collapsed step.
|
|
@@ -210,13 +269,15 @@ module Fp
|
|
|
210
269
|
end
|
|
211
270
|
|
|
212
271
|
# Bind this report's testcases to their fp markers, locally, in the checkout.
|
|
213
|
-
def resolve_results(path, opts)
|
|
272
|
+
def resolve_results(path, opts, commit_sha:, repo:)
|
|
214
273
|
testcases = Fp::JUnit.parse_file(path)
|
|
215
274
|
resolved = Fp::JUnit.ci_results(
|
|
216
275
|
testcases,
|
|
217
276
|
base_dir: opts[:base_dir] || Dir.pwd,
|
|
218
277
|
default_surface: opts[:surface],
|
|
219
|
-
default_test_type: opts[:test_type]
|
|
278
|
+
default_test_type: opts[:test_type],
|
|
279
|
+
commit_sha: commit_sha,
|
|
280
|
+
repo: repo
|
|
220
281
|
)
|
|
221
282
|
|
|
222
283
|
{
|
|
@@ -233,6 +294,25 @@ module Fp
|
|
|
233
294
|
failure_count: r.failure_count
|
|
234
295
|
}
|
|
235
296
|
end,
|
|
297
|
+
test_revisions: resolved[:test_revisions].map do |revision|
|
|
298
|
+
{
|
|
299
|
+
slug: revision.slug,
|
|
300
|
+
surface: revision.surface,
|
|
301
|
+
test_type: revision.test_type,
|
|
302
|
+
testcase_key: revision.testcase_key,
|
|
303
|
+
testcase_name: revision.testcase_name,
|
|
304
|
+
classname: revision.classname,
|
|
305
|
+
outcome: revision.outcome,
|
|
306
|
+
repo: revision.repo,
|
|
307
|
+
source_file: revision.file,
|
|
308
|
+
marker_line: revision.marker_line,
|
|
309
|
+
source_start_line: revision.start_line,
|
|
310
|
+
source_end_line: revision.end_line,
|
|
311
|
+
commit_sha: revision.commit_sha,
|
|
312
|
+
fingerprint: revision.fingerprint,
|
|
313
|
+
fingerprint_scope: revision.fingerprint_scope
|
|
314
|
+
}
|
|
315
|
+
end,
|
|
236
316
|
unmatched: resolved[:unmatched].length,
|
|
237
317
|
# Loose ends: failing tests with no fp:<slug> marker. Shipped to the server so a
|
|
238
318
|
# human can see them on the matrix; a red test nobody tracks against a
|
|
@@ -251,6 +331,27 @@ module Fp
|
|
|
251
331
|
exit 1
|
|
252
332
|
end
|
|
253
333
|
|
|
334
|
+
def resolve_commit_sha(opts)
|
|
335
|
+
explicit = presence(opts[:sha])
|
|
336
|
+
return explicit if explicit
|
|
337
|
+
|
|
338
|
+
root = Fp::JUnit.repo_root_for(opts[:base_dir] || Dir.pwd)
|
|
339
|
+
stdout, _stderr, status = Open3.capture3('git', 'rev-parse', 'HEAD', chdir: root || Dir.pwd)
|
|
340
|
+
sha = presence(stdout)
|
|
341
|
+
return sha if status.success? && sha
|
|
342
|
+
|
|
343
|
+
ci_sha = presence(ENV['GITHUB_SHA'])
|
|
344
|
+
return ci_sha if ci_sha
|
|
345
|
+
|
|
346
|
+
output.error('Could not determine the tested commit. Pass --sha <commit>.')
|
|
347
|
+
exit 1
|
|
348
|
+
end
|
|
349
|
+
|
|
350
|
+
def presence(value)
|
|
351
|
+
text = value.to_s.strip
|
|
352
|
+
text.empty? ? nil : text
|
|
353
|
+
end
|
|
354
|
+
|
|
254
355
|
# Pull the human-readable summary together. `markers_resolved` is worth printing
|
|
255
356
|
# even when everything worked: it's the number that used to silently be zero.
|
|
256
357
|
def emit_summary(opts, totals)
|
|
@@ -265,6 +366,9 @@ module Fp
|
|
|
265
366
|
markers_resolved: totals[:resolved],
|
|
266
367
|
unmatched_testcases: totals[:unmatched],
|
|
267
368
|
unmarked_failures: totals[:unmarked_failures],
|
|
369
|
+
test_revisions_reported: totals[:test_revisions],
|
|
370
|
+
changed_test_findings_opened: totals[:findings_opened],
|
|
371
|
+
bound_test_baselines_accepted: totals[:baselines_accepted],
|
|
268
372
|
skipped_unknown: totals[:unknown].uniq,
|
|
269
373
|
reports: totals[:reports]
|
|
270
374
|
}
|
|
@@ -288,6 +392,7 @@ module Fp
|
|
|
288
392
|
puts " loose ends: #{totals[:unmarked_failures].length} failing test(s) with no " \
|
|
289
393
|
'fp:<slug> marker (shown on the matrix, not a gate failure)'
|
|
290
394
|
end
|
|
395
|
+
puts " source receipts: #{totals[:test_revisions]} changed-test findings opened: #{totals[:findings_opened]}"
|
|
291
396
|
puts " created: #{totals[:created]} updated: #{totals[:updated]} marked failing (missing): #{totals[:missing_failing]}"
|
|
292
397
|
end
|
|
293
398
|
end
|
|
@@ -0,0 +1,99 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Fp
|
|
4
|
+
module Commands
|
|
5
|
+
# `fp comment <slug> --body "..."` — thread a comment onto a requirement's open
|
|
6
|
+
# change proposal.
|
|
7
|
+
#
|
|
8
|
+
# The single-open-proposal rule stops a second agent opening a rival change (two
|
|
9
|
+
# "what it was" snapshots would make a rollback ambiguous). This is the way out of
|
|
10
|
+
# that dead end: instead of a rival proposal, an agent states its case *on* the
|
|
11
|
+
# proposal a human is about to review. The comment is append-only — it never touches
|
|
12
|
+
# the requirement's status or the proposal's snapshot — so the invariant the 409
|
|
13
|
+
# protects stays intact.
|
|
14
|
+
class Comment < Base
|
|
15
|
+
KNOWN_FLAGS = {
|
|
16
|
+
project: :string, body: :string, change: :string,
|
|
17
|
+
source_type: :string, source_url: :string, source_context: :string
|
|
18
|
+
}.freeze
|
|
19
|
+
|
|
20
|
+
VALID_SOURCE_TYPES = %w[fizzy discord github_issue jira notion manual ai_chat].freeze
|
|
21
|
+
|
|
22
|
+
def run(args)
|
|
23
|
+
opts, positional = parse_flags(args, KNOWN_FLAGS)
|
|
24
|
+
require_project!(opts)
|
|
25
|
+
|
|
26
|
+
slug = positional.first
|
|
27
|
+
unless slug
|
|
28
|
+
output.error('Requirement slug is required. Usage: fp comment <slug> --body "..." [--project <project>]')
|
|
29
|
+
exit 1
|
|
30
|
+
end
|
|
31
|
+
|
|
32
|
+
body = opts[:body]
|
|
33
|
+
if body.nil? || body.strip.empty?
|
|
34
|
+
output.error('A comment body is required. Pass --body "...".')
|
|
35
|
+
exit 1
|
|
36
|
+
end
|
|
37
|
+
|
|
38
|
+
validate_source_type!(opts)
|
|
39
|
+
|
|
40
|
+
project_id = resolve_project_id(opts[:project])
|
|
41
|
+
requirement = find_requirement!(project_id, slug)
|
|
42
|
+
|
|
43
|
+
change_id = opts[:change] || requirement['pending_change_id']
|
|
44
|
+
if change_id.nil? || change_id.to_s.empty?
|
|
45
|
+
output.error("Requirement '#{slug}' has no open proposal to comment on.")
|
|
46
|
+
exit 1
|
|
47
|
+
end
|
|
48
|
+
|
|
49
|
+
result = client.comment_requirement_change(requirement['id'], change_id, build_params(opts))
|
|
50
|
+
unless result[:ok]
|
|
51
|
+
output.error(result[:error], status: result[:status])
|
|
52
|
+
exit 1
|
|
53
|
+
end
|
|
54
|
+
|
|
55
|
+
print_result(result[:data], slug: slug)
|
|
56
|
+
end
|
|
57
|
+
|
|
58
|
+
private
|
|
59
|
+
|
|
60
|
+
def validate_source_type!(opts)
|
|
61
|
+
return unless opts[:source_type] && !VALID_SOURCE_TYPES.include?(opts[:source_type])
|
|
62
|
+
|
|
63
|
+
output.error("Invalid source type '#{opts[:source_type]}'. Must be one of: #{VALID_SOURCE_TYPES.join(', ')}")
|
|
64
|
+
exit 1
|
|
65
|
+
end
|
|
66
|
+
|
|
67
|
+
def find_requirement!(project_id, slug)
|
|
68
|
+
result = client.find_requirement_by_slug(project_id, slug)
|
|
69
|
+
unless result[:ok]
|
|
70
|
+
output.error(result[:error], status: result[:status])
|
|
71
|
+
exit 1
|
|
72
|
+
end
|
|
73
|
+
result[:data]['requirement']
|
|
74
|
+
end
|
|
75
|
+
|
|
76
|
+
def build_params(opts)
|
|
77
|
+
params = { body: opts[:body] }
|
|
78
|
+
params[:source_type] = opts[:source_type] if opts[:source_type]
|
|
79
|
+
params[:source_url] = opts[:source_url] if opts[:source_url]
|
|
80
|
+
params[:source_context] = opts[:source_context] if opts[:source_context]
|
|
81
|
+
params
|
|
82
|
+
end
|
|
83
|
+
|
|
84
|
+
def print_result(data, slug:)
|
|
85
|
+
change = data['change'] || {}
|
|
86
|
+
|
|
87
|
+
output.success(data) do
|
|
88
|
+
puts "Commented on the open proposal for '#{slug}'."
|
|
89
|
+
puts
|
|
90
|
+
puts " Comment: #{change['body']}"
|
|
91
|
+
puts " On: #{change['proposal_id']}"
|
|
92
|
+
puts
|
|
93
|
+
puts 'A human reviewing the proposal will see this comment in the history.'
|
|
94
|
+
puts "Run `fp history #{slug} --project <project>` to see the thread."
|
|
95
|
+
end
|
|
96
|
+
end
|
|
97
|
+
end
|
|
98
|
+
end
|
|
99
|
+
end
|
data/lib/fp/commands/help.rb
CHANGED
|
@@ -22,6 +22,7 @@ module Fp
|
|
|
22
22
|
propose --project <slug> ... Propose a new requirement (as draft)
|
|
23
23
|
propose-change <slug> ... Propose a change to an existing requirement
|
|
24
24
|
propose-change <slug> --removal Propose deleting a requirement
|
|
25
|
+
comment <slug> --body "..." Comment on a requirement's open change proposal
|
|
25
26
|
history <slug> --project <slug> Show a requirement's change history + diffs
|
|
26
27
|
changes [--source-url URL] [--github-pr] Show or publish work-item-linked activity
|
|
27
28
|
attach <slug> <file> Upload a file attachment to a requirement
|
|
@@ -33,6 +34,8 @@ module Fp
|
|
|
33
34
|
report --junit <file> ... Upload evidence from a JUnit XML report (no CI)
|
|
34
35
|
ci-report --junit <file> ... Ingest CI test results (passing/failing) — CI only
|
|
35
36
|
ci-report --junit <file> --pr URL Ingest a PR's CI results as pr_passing — CI only
|
|
37
|
+
changed-tests List unresolved bound-test revision findings
|
|
38
|
+
changed-tests assess <id> ... Explain whether an exact revision still proves its requirement
|
|
36
39
|
check Enforce parity; exits non-zero on violations
|
|
37
40
|
matrix --project <slug> Show the parity matrix
|
|
38
41
|
export --project <slug> Export all requirements as a Markdown document
|
|
@@ -77,6 +80,24 @@ module Fp
|
|
|
77
80
|
merged-green truth. Run ci-report *without* --pr on a push to the default branch to
|
|
78
81
|
record the real `passing`, which supersedes any pr_passing a prior PR left behind.
|
|
79
82
|
|
|
83
|
+
CHANGED BOUND TESTS
|
|
84
|
+
`fp ci-report` fingerprints every bound testcase at the checkout commit. It uses
|
|
85
|
+
an isolated-test SHA-256 when the source boundary is reliable and labels a whole-file
|
|
86
|
+
fallback otherwise. A different digest opens an audit finding without turning a green
|
|
87
|
+
test red or replacing the accepted baseline.
|
|
88
|
+
|
|
89
|
+
fp changed-tests
|
|
90
|
+
fp changed-tests --json
|
|
91
|
+
fp changed-tests assess <finding-id> \
|
|
92
|
+
--candidate-fingerprint <sha256> \
|
|
93
|
+
--classification still_proves \
|
|
94
|
+
--explanation "The setup changed; the same observable result is asserted."
|
|
95
|
+
|
|
96
|
+
Classifications are still_proves, no_longer_proves, and uncertain. Only still_proves
|
|
97
|
+
resolves that exact fingerprint; the other two remain open for a human product decision.
|
|
98
|
+
A stale or later candidate needs a new assessment. A PR candidate becomes accepted only
|
|
99
|
+
when a default-branch CI report later observes the same fingerprint.
|
|
100
|
+
|
|
80
101
|
AUTHENTICATION
|
|
81
102
|
Set FP_API_KEY environment variable, or use a profile:
|
|
82
103
|
|
|
@@ -175,6 +196,18 @@ module Fp
|
|
|
175
196
|
# which promise reflects product intent and record that decision with propose-change
|
|
176
197
|
# (or --removal). Never weaken the test or add a requirement around the conflict;
|
|
177
198
|
# ask a human if intent is unclear.
|
|
199
|
+
# Existing tests are protected product evidence. Flag any edit to a test backing
|
|
200
|
+
# passing evidence and get explicit human approval before changing its expected
|
|
201
|
+
# behaviour. When FeatureParity reports a changed bound test, run `fp changed-tests`
|
|
202
|
+
# (or add `--json`), compare the accepted and candidate revisions with the requirement,
|
|
203
|
+
# then assess the exact candidate fingerprint with a plain-language explanation as
|
|
204
|
+
# `still_proves`, `no_longer_proves`, or `uncertain`. Only still_proves resolves that
|
|
205
|
+
# fingerprint; possible behaviour changes remain open for a human decision, and a later
|
|
206
|
+
# digest needs a new assessment. This audit finding is separate from the CI outcome and
|
|
207
|
+
# from `suspect`, which means the requirement wording changed.
|
|
208
|
+
# A new requirement — draft or active — does not supersede a pre-existing failing test;
|
|
209
|
+
# surface the conflict and do not change or adopt that test as proof without explicit
|
|
210
|
+
# human approval.
|
|
178
211
|
#
|
|
179
212
|
# Write the requirement as a product decision someone can approve, not a design
|
|
180
213
|
# task. It should sound like a capable teammate, not AI boilerplate.
|