featureparity 0.0.15 → 0.0.17
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/fp/cli.rb +3 -0
- data/lib/fp/client.rb +51 -3
- data/lib/fp/commands/changed_tests.rb +126 -0
- data/lib/fp/commands/changes.rb +359 -0
- data/lib/fp/commands/ci_report.rb +133 -28
- data/lib/fp/commands/comment.rb +99 -0
- data/lib/fp/commands/help.rb +34 -0
- data/lib/fp/commands/init.rb +71 -12
- data/lib/fp/commands/propose_change.rb +46 -1
- data/lib/fp/commands/report.rb +63 -6
- data/lib/fp/commands/show.rb +20 -2
- data/lib/fp/commands.rb +3 -0
- data/lib/fp/junit.rb +50 -5
- data/lib/fp/source_fingerprint.rb +193 -0
- data/lib/fp/version.rb +1 -1
- data/lib/fp.rb +1 -0
- data/skills/fp/SKILL.md +33 -0
- data/skills/fp/references/cli.md +86 -1
- metadata +6 -2
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
3
|
require 'set'
|
|
4
|
+
require 'open3'
|
|
4
5
|
|
|
5
6
|
module Fp
|
|
6
7
|
module Commands
|
|
@@ -34,8 +35,10 @@ module Fp
|
|
|
34
35
|
# `fp report`.
|
|
35
36
|
#
|
|
36
37
|
# Multiple --junit flags may be passed to ingest several reports for the same
|
|
37
|
-
# surface in one invocation (e.g.
|
|
38
|
-
# uploaded
|
|
38
|
+
# surface in one invocation (e.g. unit and e2e suites, or test shards). They are
|
|
39
|
+
# resolved and uploaded as one run. This is load-bearing: each API ingest performs
|
|
40
|
+
# the vanished-test sweep, so uploading partitions independently would make every
|
|
41
|
+
# partition call the passing tests in its siblings "missing".
|
|
39
42
|
#
|
|
40
43
|
# --base-dir tells the CLI where the report's relative test-file paths resolve from
|
|
41
44
|
# (vitest, for instance, reports paths relative to the frontend package, not the repo
|
|
@@ -55,6 +58,8 @@ module Fp
|
|
|
55
58
|
pr: :string,
|
|
56
59
|
junit: :string,
|
|
57
60
|
base_dir: :string,
|
|
61
|
+
repo: :string,
|
|
62
|
+
sha: :string,
|
|
58
63
|
allow_empty: :boolean
|
|
59
64
|
}.freeze
|
|
60
65
|
|
|
@@ -81,42 +86,64 @@ module Fp
|
|
|
81
86
|
|
|
82
87
|
project_id = resolve_project_id(opts[:project])
|
|
83
88
|
known = known_slugs(project_id)
|
|
89
|
+
commit_sha = resolve_commit_sha(opts)
|
|
90
|
+
repo = opts[:repo] || presence(ENV['GITHUB_REPOSITORY'])
|
|
84
91
|
|
|
85
92
|
totals = { created: 0, updated: 0, missing_failing: 0, reports: [], resolved: 0,
|
|
86
|
-
unmatched: 0, unknown: [], unmarked_failures: []
|
|
93
|
+
unmatched: 0, unknown: [], unmarked_failures: [], test_revisions: 0,
|
|
94
|
+
findings_opened: 0, baselines_accepted: 0 }
|
|
87
95
|
|
|
88
|
-
junit_paths.
|
|
89
|
-
|
|
90
|
-
resolved = resolve_results(path, opts)
|
|
96
|
+
resolved_reports = junit_paths.map do |path|
|
|
97
|
+
resolved = resolve_results(path, opts, commit_sha: commit_sha, repo: repo)
|
|
91
98
|
totals[:unmatched] += resolved[:unmatched]
|
|
92
99
|
|
|
93
100
|
results, unknown = partition_known(resolved[:results], known)
|
|
101
|
+
test_revisions, revision_unknown = partition_known(resolved[:test_revisions], known)
|
|
94
102
|
totals[:resolved] += results.length
|
|
95
|
-
totals[:unknown].concat(unknown)
|
|
103
|
+
totals[:unknown].concat(unknown).concat(revision_unknown)
|
|
96
104
|
totals[:unmarked_failures].concat(resolved[:unmarked_failures])
|
|
105
|
+
totals[:test_revisions] += test_revisions.length
|
|
97
106
|
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
107
|
+
{ path: path, xml: File.read(path), results: results, test_revisions: test_revisions }
|
|
108
|
+
end
|
|
109
|
+
|
|
110
|
+
# One surface run may be split across several reports (unit + e2e, shards, etc.).
|
|
111
|
+
# The API's vanished-test sweep must see their union in one request; posting each
|
|
112
|
+
# partition separately makes the tests in every sibling report appear to have
|
|
113
|
+
# disappeared. Duplicate slug/surface/type claims are collapsed conservatively.
|
|
114
|
+
combined_results = combine_results(resolved_reports.flat_map { |report| report[:results] })
|
|
115
|
+
combined_revisions = combine_test_revisions(
|
|
116
|
+
resolved_reports.flat_map { |report| report[:test_revisions] }
|
|
117
|
+
)
|
|
118
|
+
result = client.ingest_ci_results(
|
|
119
|
+
project_id,
|
|
120
|
+
ci_url: opts[:ci_url],
|
|
121
|
+
surface: opts[:surface],
|
|
122
|
+
# Resolved results are authoritative. Keep one valid raw report for compatibility
|
|
123
|
+
# with servers old enough to ignore client-resolved results.
|
|
124
|
+
junit_xml: resolved_reports.first[:xml],
|
|
125
|
+
pr_url: opts[:pr],
|
|
126
|
+
results: combined_results,
|
|
127
|
+
test_revisions: combined_revisions,
|
|
128
|
+
unmarked_failures: totals[:unmarked_failures]
|
|
129
|
+
)
|
|
112
130
|
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
totals[:missing_failing] += data['missing_marked_failing'].to_i
|
|
117
|
-
totals[:reports] << { path: path, results: data['results'] || [] }
|
|
131
|
+
unless result[:ok]
|
|
132
|
+
output.error("Failed to ingest #{junit_paths.join(', ')}: #{result[:error]}", status: result[:status])
|
|
133
|
+
exit 1
|
|
118
134
|
end
|
|
119
135
|
|
|
136
|
+
data = result[:data] || {}
|
|
137
|
+
# Belt's implicit instance-variable serializer wrapped this response in older
|
|
138
|
+
# deployments. Accept both shapes while the API now returns the result directly.
|
|
139
|
+
data = data['ingest_result'] if data['ingest_result'].is_a?(Hash)
|
|
140
|
+
totals[:created] += data['evidence_created'].to_i
|
|
141
|
+
totals[:updated] += data['evidence_updated'].to_i
|
|
142
|
+
totals[:missing_failing] += data['missing_marked_failing'].to_i
|
|
143
|
+
totals[:findings_opened] += data['changed_test_findings_opened'].to_i
|
|
144
|
+
totals[:baselines_accepted] += data['bound_test_baselines_accepted'].to_i
|
|
145
|
+
totals[:reports] << { paths: junit_paths, results: data['results'] || [] }
|
|
146
|
+
|
|
120
147
|
# Reporting nothing is the failure this command exists to make visible: a run that
|
|
121
148
|
# resolves no markers has told the matrix nothing, and used to do so silently.
|
|
122
149
|
if totals[:resolved].zero? && !opts[:allow_empty]
|
|
@@ -194,6 +221,29 @@ module Fp
|
|
|
194
221
|
(result[:data]['requirements'] || []).map { |r| r['slug'] }.compact.to_set
|
|
195
222
|
end
|
|
196
223
|
|
|
224
|
+
# Collapse the same marker key across report partitions before uploading it. A red
|
|
225
|
+
# claim wins; otherwise any executed passing test wins over an all-skipped stub.
|
|
226
|
+
# Counts are summed so the evidence still describes the complete surface run, and
|
|
227
|
+
# the source points at the claim that determined the verdict.
|
|
228
|
+
def combine_results(results)
|
|
229
|
+
severity = { 'failing' => 0, 'passing' => 1, 'stub' => 2 }
|
|
230
|
+
|
|
231
|
+
results.each_with_object({}) do |result, combined|
|
|
232
|
+
key = [result[:slug], result[:surface], result[:test_type]]
|
|
233
|
+
current = combined[key]
|
|
234
|
+
unless current
|
|
235
|
+
combined[key] = result.dup
|
|
236
|
+
next
|
|
237
|
+
end
|
|
238
|
+
|
|
239
|
+
decisive = [current, result].min_by { |entry| severity.fetch(entry[:status], 0) }
|
|
240
|
+
merged = decisive.dup
|
|
241
|
+
merged[:test_count] = current[:test_count].to_i + result[:test_count].to_i
|
|
242
|
+
merged[:failure_count] = current[:failure_count].to_i + result[:failure_count].to_i
|
|
243
|
+
combined[key] = merged
|
|
244
|
+
end.values
|
|
245
|
+
end
|
|
246
|
+
|
|
197
247
|
def partition_known(results, known)
|
|
198
248
|
return [results, []] if known.nil?
|
|
199
249
|
|
|
@@ -201,6 +251,15 @@ module Fp
|
|
|
201
251
|
[kept, unknown.map { |r| r[:slug] }.uniq]
|
|
202
252
|
end
|
|
203
253
|
|
|
254
|
+
# Reports can be split into shards. Keep every independently identified testcase,
|
|
255
|
+
# but collapse an exact duplicate emitted by overlapping reports.
|
|
256
|
+
def combine_test_revisions(revisions)
|
|
257
|
+
revisions.each_with_object({}) do |revision, combined|
|
|
258
|
+
key = [revision[:slug], revision[:surface], revision[:test_type], revision[:testcase_key]]
|
|
259
|
+
combined[key] = revision
|
|
260
|
+
end.values
|
|
261
|
+
end
|
|
262
|
+
|
|
204
263
|
# A warning that survives a CI log. Stderr always; a GitHub Actions annotation too
|
|
205
264
|
# when we're running in one, so it shows up in the run summary rather than scrolling
|
|
206
265
|
# past in a collapsed step.
|
|
@@ -210,13 +269,15 @@ module Fp
|
|
|
210
269
|
end
|
|
211
270
|
|
|
212
271
|
# Bind this report's testcases to their fp markers, locally, in the checkout.
|
|
213
|
-
def resolve_results(path, opts)
|
|
272
|
+
def resolve_results(path, opts, commit_sha:, repo:)
|
|
214
273
|
testcases = Fp::JUnit.parse_file(path)
|
|
215
274
|
resolved = Fp::JUnit.ci_results(
|
|
216
275
|
testcases,
|
|
217
276
|
base_dir: opts[:base_dir] || Dir.pwd,
|
|
218
277
|
default_surface: opts[:surface],
|
|
219
|
-
default_test_type: opts[:test_type]
|
|
278
|
+
default_test_type: opts[:test_type],
|
|
279
|
+
commit_sha: commit_sha,
|
|
280
|
+
repo: repo
|
|
220
281
|
)
|
|
221
282
|
|
|
222
283
|
{
|
|
@@ -233,6 +294,25 @@ module Fp
|
|
|
233
294
|
failure_count: r.failure_count
|
|
234
295
|
}
|
|
235
296
|
end,
|
|
297
|
+
test_revisions: resolved[:test_revisions].map do |revision|
|
|
298
|
+
{
|
|
299
|
+
slug: revision.slug,
|
|
300
|
+
surface: revision.surface,
|
|
301
|
+
test_type: revision.test_type,
|
|
302
|
+
testcase_key: revision.testcase_key,
|
|
303
|
+
testcase_name: revision.testcase_name,
|
|
304
|
+
classname: revision.classname,
|
|
305
|
+
outcome: revision.outcome,
|
|
306
|
+
repo: revision.repo,
|
|
307
|
+
source_file: revision.file,
|
|
308
|
+
marker_line: revision.marker_line,
|
|
309
|
+
source_start_line: revision.start_line,
|
|
310
|
+
source_end_line: revision.end_line,
|
|
311
|
+
commit_sha: revision.commit_sha,
|
|
312
|
+
fingerprint: revision.fingerprint,
|
|
313
|
+
fingerprint_scope: revision.fingerprint_scope
|
|
314
|
+
}
|
|
315
|
+
end,
|
|
236
316
|
unmatched: resolved[:unmatched].length,
|
|
237
317
|
# Loose ends: failing tests with no fp:<slug> marker. Shipped to the server so a
|
|
238
318
|
# human can see them on the matrix; a red test nobody tracks against a
|
|
@@ -251,6 +331,27 @@ module Fp
|
|
|
251
331
|
exit 1
|
|
252
332
|
end
|
|
253
333
|
|
|
334
|
+
def resolve_commit_sha(opts)
|
|
335
|
+
explicit = presence(opts[:sha])
|
|
336
|
+
return explicit if explicit
|
|
337
|
+
|
|
338
|
+
root = Fp::JUnit.repo_root_for(opts[:base_dir] || Dir.pwd)
|
|
339
|
+
stdout, _stderr, status = Open3.capture3('git', 'rev-parse', 'HEAD', chdir: root || Dir.pwd)
|
|
340
|
+
sha = presence(stdout)
|
|
341
|
+
return sha if status.success? && sha
|
|
342
|
+
|
|
343
|
+
ci_sha = presence(ENV['GITHUB_SHA'])
|
|
344
|
+
return ci_sha if ci_sha
|
|
345
|
+
|
|
346
|
+
output.error('Could not determine the tested commit. Pass --sha <commit>.')
|
|
347
|
+
exit 1
|
|
348
|
+
end
|
|
349
|
+
|
|
350
|
+
def presence(value)
|
|
351
|
+
text = value.to_s.strip
|
|
352
|
+
text.empty? ? nil : text
|
|
353
|
+
end
|
|
354
|
+
|
|
254
355
|
# Pull the human-readable summary together. `markers_resolved` is worth printing
|
|
255
356
|
# even when everything worked: it's the number that used to silently be zero.
|
|
256
357
|
def emit_summary(opts, totals)
|
|
@@ -265,6 +366,9 @@ module Fp
|
|
|
265
366
|
markers_resolved: totals[:resolved],
|
|
266
367
|
unmatched_testcases: totals[:unmatched],
|
|
267
368
|
unmarked_failures: totals[:unmarked_failures],
|
|
369
|
+
test_revisions_reported: totals[:test_revisions],
|
|
370
|
+
changed_test_findings_opened: totals[:findings_opened],
|
|
371
|
+
bound_test_baselines_accepted: totals[:baselines_accepted],
|
|
268
372
|
skipped_unknown: totals[:unknown].uniq,
|
|
269
373
|
reports: totals[:reports]
|
|
270
374
|
}
|
|
@@ -288,6 +392,7 @@ module Fp
|
|
|
288
392
|
puts " loose ends: #{totals[:unmarked_failures].length} failing test(s) with no " \
|
|
289
393
|
'fp:<slug> marker (shown on the matrix, not a gate failure)'
|
|
290
394
|
end
|
|
395
|
+
puts " source receipts: #{totals[:test_revisions]} changed-test findings opened: #{totals[:findings_opened]}"
|
|
291
396
|
puts " created: #{totals[:created]} updated: #{totals[:updated]} marked failing (missing): #{totals[:missing_failing]}"
|
|
292
397
|
end
|
|
293
398
|
end
|
|
@@ -0,0 +1,99 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Fp
|
|
4
|
+
module Commands
|
|
5
|
+
# `fp comment <slug> --body "..."` — thread a comment onto a requirement's open
|
|
6
|
+
# change proposal.
|
|
7
|
+
#
|
|
8
|
+
# The single-open-proposal rule stops a second agent opening a rival change (two
|
|
9
|
+
# "what it was" snapshots would make a rollback ambiguous). This is the way out of
|
|
10
|
+
# that dead end: instead of a rival proposal, an agent states its case *on* the
|
|
11
|
+
# proposal a human is about to review. The comment is append-only — it never touches
|
|
12
|
+
# the requirement's status or the proposal's snapshot — so the invariant the 409
|
|
13
|
+
# protects stays intact.
|
|
14
|
+
class Comment < Base
|
|
15
|
+
KNOWN_FLAGS = {
|
|
16
|
+
project: :string, body: :string, change: :string,
|
|
17
|
+
source_type: :string, source_url: :string, source_context: :string
|
|
18
|
+
}.freeze
|
|
19
|
+
|
|
20
|
+
VALID_SOURCE_TYPES = %w[fizzy discord github_issue jira notion manual ai_chat].freeze
|
|
21
|
+
|
|
22
|
+
def run(args)
|
|
23
|
+
opts, positional = parse_flags(args, KNOWN_FLAGS)
|
|
24
|
+
require_project!(opts)
|
|
25
|
+
|
|
26
|
+
slug = positional.first
|
|
27
|
+
unless slug
|
|
28
|
+
output.error('Requirement slug is required. Usage: fp comment <slug> --body "..." [--project <project>]')
|
|
29
|
+
exit 1
|
|
30
|
+
end
|
|
31
|
+
|
|
32
|
+
body = opts[:body]
|
|
33
|
+
if body.nil? || body.strip.empty?
|
|
34
|
+
output.error('A comment body is required. Pass --body "...".')
|
|
35
|
+
exit 1
|
|
36
|
+
end
|
|
37
|
+
|
|
38
|
+
validate_source_type!(opts)
|
|
39
|
+
|
|
40
|
+
project_id = resolve_project_id(opts[:project])
|
|
41
|
+
requirement = find_requirement!(project_id, slug)
|
|
42
|
+
|
|
43
|
+
change_id = opts[:change] || requirement['pending_change_id']
|
|
44
|
+
if change_id.nil? || change_id.to_s.empty?
|
|
45
|
+
output.error("Requirement '#{slug}' has no open proposal to comment on.")
|
|
46
|
+
exit 1
|
|
47
|
+
end
|
|
48
|
+
|
|
49
|
+
result = client.comment_requirement_change(requirement['id'], change_id, build_params(opts))
|
|
50
|
+
unless result[:ok]
|
|
51
|
+
output.error(result[:error], status: result[:status])
|
|
52
|
+
exit 1
|
|
53
|
+
end
|
|
54
|
+
|
|
55
|
+
print_result(result[:data], slug: slug)
|
|
56
|
+
end
|
|
57
|
+
|
|
58
|
+
private
|
|
59
|
+
|
|
60
|
+
def validate_source_type!(opts)
|
|
61
|
+
return unless opts[:source_type] && !VALID_SOURCE_TYPES.include?(opts[:source_type])
|
|
62
|
+
|
|
63
|
+
output.error("Invalid source type '#{opts[:source_type]}'. Must be one of: #{VALID_SOURCE_TYPES.join(', ')}")
|
|
64
|
+
exit 1
|
|
65
|
+
end
|
|
66
|
+
|
|
67
|
+
def find_requirement!(project_id, slug)
|
|
68
|
+
result = client.find_requirement_by_slug(project_id, slug)
|
|
69
|
+
unless result[:ok]
|
|
70
|
+
output.error(result[:error], status: result[:status])
|
|
71
|
+
exit 1
|
|
72
|
+
end
|
|
73
|
+
result[:data]['requirement']
|
|
74
|
+
end
|
|
75
|
+
|
|
76
|
+
def build_params(opts)
|
|
77
|
+
params = { body: opts[:body] }
|
|
78
|
+
params[:source_type] = opts[:source_type] if opts[:source_type]
|
|
79
|
+
params[:source_url] = opts[:source_url] if opts[:source_url]
|
|
80
|
+
params[:source_context] = opts[:source_context] if opts[:source_context]
|
|
81
|
+
params
|
|
82
|
+
end
|
|
83
|
+
|
|
84
|
+
def print_result(data, slug:)
|
|
85
|
+
change = data['change'] || {}
|
|
86
|
+
|
|
87
|
+
output.success(data) do
|
|
88
|
+
puts "Commented on the open proposal for '#{slug}'."
|
|
89
|
+
puts
|
|
90
|
+
puts " Comment: #{change['body']}"
|
|
91
|
+
puts " On: #{change['proposal_id']}"
|
|
92
|
+
puts
|
|
93
|
+
puts 'A human reviewing the proposal will see this comment in the history.'
|
|
94
|
+
puts "Run `fp history #{slug} --project <project>` to see the thread."
|
|
95
|
+
end
|
|
96
|
+
end
|
|
97
|
+
end
|
|
98
|
+
end
|
|
99
|
+
end
|
data/lib/fp/commands/help.rb
CHANGED
|
@@ -22,7 +22,9 @@ module Fp
|
|
|
22
22
|
propose --project <slug> ... Propose a new requirement (as draft)
|
|
23
23
|
propose-change <slug> ... Propose a change to an existing requirement
|
|
24
24
|
propose-change <slug> --removal Propose deleting a requirement
|
|
25
|
+
comment <slug> --body "..." Comment on a requirement's open change proposal
|
|
25
26
|
history <slug> --project <slug> Show a requirement's change history + diffs
|
|
27
|
+
changes [--source-url URL] [--github-pr] Show or publish work-item-linked activity
|
|
26
28
|
attach <slug> <file> Upload a file attachment to a requirement
|
|
27
29
|
attachments <slug> List attachments for a requirement
|
|
28
30
|
attachments <slug> <id> Get download URL for an attachment
|
|
@@ -32,6 +34,8 @@ module Fp
|
|
|
32
34
|
report --junit <file> ... Upload evidence from a JUnit XML report (no CI)
|
|
33
35
|
ci-report --junit <file> ... Ingest CI test results (passing/failing) — CI only
|
|
34
36
|
ci-report --junit <file> --pr URL Ingest a PR's CI results as pr_passing — CI only
|
|
37
|
+
changed-tests List unresolved bound-test revision findings
|
|
38
|
+
changed-tests assess <id> ... Explain whether an exact revision still proves its requirement
|
|
35
39
|
check Enforce parity; exits non-zero on violations
|
|
36
40
|
matrix --project <slug> Show the parity matrix
|
|
37
41
|
export --project <slug> Export all requirements as a Markdown document
|
|
@@ -76,6 +80,24 @@ module Fp
|
|
|
76
80
|
merged-green truth. Run ci-report *without* --pr on a push to the default branch to
|
|
77
81
|
record the real `passing`, which supersedes any pr_passing a prior PR left behind.
|
|
78
82
|
|
|
83
|
+
CHANGED BOUND TESTS
|
|
84
|
+
`fp ci-report` fingerprints every bound testcase at the checkout commit. It uses
|
|
85
|
+
an isolated-test SHA-256 when the source boundary is reliable and labels a whole-file
|
|
86
|
+
fallback otherwise. A different digest opens an audit finding without turning a green
|
|
87
|
+
test red or replacing the accepted baseline.
|
|
88
|
+
|
|
89
|
+
fp changed-tests
|
|
90
|
+
fp changed-tests --json
|
|
91
|
+
fp changed-tests assess <finding-id> \
|
|
92
|
+
--candidate-fingerprint <sha256> \
|
|
93
|
+
--classification still_proves \
|
|
94
|
+
--explanation "The setup changed; the same observable result is asserted."
|
|
95
|
+
|
|
96
|
+
Classifications are still_proves, no_longer_proves, and uncertain. Only still_proves
|
|
97
|
+
resolves that exact fingerprint; the other two remain open for a human product decision.
|
|
98
|
+
A stale or later candidate needs a new assessment. A PR candidate becomes accepted only
|
|
99
|
+
when a default-branch CI report later observes the same fingerprint.
|
|
100
|
+
|
|
79
101
|
AUTHENTICATION
|
|
80
102
|
Set FP_API_KEY environment variable, or use a profile:
|
|
81
103
|
|
|
@@ -174,6 +196,18 @@ module Fp
|
|
|
174
196
|
# which promise reflects product intent and record that decision with propose-change
|
|
175
197
|
# (or --removal). Never weaken the test or add a requirement around the conflict;
|
|
176
198
|
# ask a human if intent is unclear.
|
|
199
|
+
# Existing tests are protected product evidence. Flag any edit to a test backing
|
|
200
|
+
# passing evidence and get explicit human approval before changing its expected
|
|
201
|
+
# behaviour. When FeatureParity reports a changed bound test, run `fp changed-tests`
|
|
202
|
+
# (or add `--json`), compare the accepted and candidate revisions with the requirement,
|
|
203
|
+
# then assess the exact candidate fingerprint with a plain-language explanation as
|
|
204
|
+
# `still_proves`, `no_longer_proves`, or `uncertain`. Only still_proves resolves that
|
|
205
|
+
# fingerprint; possible behaviour changes remain open for a human decision, and a later
|
|
206
|
+
# digest needs a new assessment. This audit finding is separate from the CI outcome and
|
|
207
|
+
# from `suspect`, which means the requirement wording changed.
|
|
208
|
+
# A new requirement — draft or active — does not supersede a pre-existing failing test;
|
|
209
|
+
# surface the conflict and do not change or adopt that test as proof without explicit
|
|
210
|
+
# human approval.
|
|
177
211
|
#
|
|
178
212
|
# Write the requirement as a product decision someone can approve, not a design
|
|
179
213
|
# task. It should sound like a capable teammate, not AI boilerplate.
|
data/lib/fp/commands/init.rb
CHANGED
|
@@ -180,17 +180,19 @@ module Fp
|
|
|
180
180
|
end
|
|
181
181
|
end
|
|
182
182
|
|
|
183
|
-
# The CI gate
|
|
184
|
-
#
|
|
185
|
-
#
|
|
183
|
+
# The CI gate and its adaptable reporting recipe are separate concerns: the gate
|
|
184
|
+
# blocks pull requests, while reporting needs a trusted context with FP_API_KEY and
|
|
185
|
+
# should cover both pull requests and default-branch pushes.
|
|
186
186
|
def workflow_body
|
|
187
187
|
<<~YAML
|
|
188
188
|
# FeatureParity — parity enforcement.
|
|
189
189
|
#
|
|
190
|
-
# Generated by `fp init`.
|
|
190
|
+
# Generated by `fp init`. The gate is ready to use; adapt and uncomment the
|
|
191
|
+
# report recipe once this repository emits JUnit:
|
|
191
192
|
#
|
|
192
|
-
# report On
|
|
193
|
-
#
|
|
193
|
+
# report On trusted pull requests, report candidate test revisions before
|
|
194
|
+
# merge. On default-branch pushes, record merged pass/fail and promote
|
|
195
|
+
# an approved exact test revision into the accepted baseline.
|
|
194
196
|
# gate On pull requests, fail the build when the project violates its
|
|
195
197
|
# enforcement level. Make this a required status check in branch
|
|
196
198
|
# protection to actually block merges.
|
|
@@ -213,6 +215,9 @@ module Fp
|
|
|
213
215
|
gate:
|
|
214
216
|
name: Parity gate
|
|
215
217
|
runs-on: ubuntu-latest
|
|
218
|
+
permissions:
|
|
219
|
+
contents: read
|
|
220
|
+
pull-requests: write
|
|
216
221
|
steps:
|
|
217
222
|
- uses: actions/checkout@v4
|
|
218
223
|
|
|
@@ -223,6 +228,21 @@ module Fp
|
|
|
223
228
|
- name: Install fp
|
|
224
229
|
run: gem install featureparity
|
|
225
230
|
|
|
231
|
+
- name: Update PR requirement changes
|
|
232
|
+
if: github.event_name == 'pull_request'
|
|
233
|
+
env:
|
|
234
|
+
FP_API_KEY: ${{ secrets.FP_API_KEY }}
|
|
235
|
+
GITHUB_TOKEN: ${{ github.token }}
|
|
236
|
+
run: |
|
|
237
|
+
if fp help | grep -q -- '--github-pr'; then
|
|
238
|
+
fp changes --github-pr --github-summary
|
|
239
|
+
else
|
|
240
|
+
echo '## FeatureParity requirement changes' >> "$GITHUB_STEP_SUMMARY"
|
|
241
|
+
echo '' >> "$GITHUB_STEP_SUMMARY"
|
|
242
|
+
echo 'The installed fp version cannot update the pull request description yet.' >> "$GITHUB_STEP_SUMMARY"
|
|
243
|
+
exit 1
|
|
244
|
+
fi
|
|
245
|
+
|
|
226
246
|
# Exits non-zero when the project violates its enforcement level.
|
|
227
247
|
# Add `--warn-only` while adopting if you need the annotations without
|
|
228
248
|
# blocking merges yet.
|
|
@@ -243,30 +263,44 @@ module Fp
|
|
|
243
263
|
run: fp check
|
|
244
264
|
|
|
245
265
|
# Uncomment and adapt once your suite emits JUnit XML. `fp ci-report` writes
|
|
246
|
-
# real passing/failing evidence
|
|
247
|
-
#
|
|
266
|
+
# real passing/failing evidence and one source fingerprint per bound testcase.
|
|
267
|
+
# Run it on trusted pull requests too: `--pr` lets FeatureParity open a changed-test
|
|
268
|
+
# finding before merge without replacing the accepted default-branch baseline.
|
|
269
|
+
# The later default-branch report records merged truth and promotes an approved exact
|
|
270
|
+
# fingerprint. Without reporting, the matrix only has agent-filed `present` markers
|
|
271
|
+
# and changed bound tests cannot be detected.
|
|
248
272
|
#
|
|
249
273
|
# It reads the fp:<slug> markers out of the checkout, so run it in the repo
|
|
250
274
|
# and pass --base-dir when the report's test-file paths are relative to a
|
|
251
275
|
# subdirectory (vitest, for instance, reports them relative to its package).
|
|
276
|
+
# Keep `if: always()` so red tests are reported. Fork pull requests cannot read
|
|
277
|
+
# FP_API_KEY and are skipped by the HAVE_FP_KEY guard.
|
|
252
278
|
#
|
|
253
279
|
# report:
|
|
254
280
|
# name: Report evidence
|
|
255
|
-
# if: github.event_name == 'push'
|
|
281
|
+
# if: github.event_name == 'push' || github.event_name == 'pull_request'
|
|
256
282
|
# runs-on: ubuntu-latest
|
|
283
|
+
# env:
|
|
284
|
+
# HAVE_FP_KEY: ${{ secrets.FP_API_KEY != '' }}
|
|
257
285
|
# steps:
|
|
258
286
|
# - uses: actions/checkout@v4
|
|
259
287
|
# - name: Run tests (emit JUnit)
|
|
260
288
|
# run: echo "replace with your test command emitting junit.xml"
|
|
261
289
|
# - name: Report to FeatureParity
|
|
262
|
-
# if: always()
|
|
290
|
+
# if: always() && env.HAVE_FP_KEY == 'true'
|
|
263
291
|
# env:
|
|
264
292
|
# FP_API_KEY: ${{ secrets.FP_API_KEY }}
|
|
265
293
|
# run: |
|
|
266
294
|
# gem install featureparity
|
|
267
|
-
#
|
|
268
|
-
# --
|
|
295
|
+
# args=(
|
|
296
|
+
# --junit junit.xml
|
|
297
|
+
# --surface <surface-key>
|
|
269
298
|
# --ci-url "${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
|
|
299
|
+
# )
|
|
300
|
+
# if [ "${{ github.event_name }}" = "pull_request" ]; then
|
|
301
|
+
# args+=(--pr "${{ github.event.pull_request.html_url }}")
|
|
302
|
+
# fi
|
|
303
|
+
# fp ci-report "${args[@]}"
|
|
270
304
|
YAML
|
|
271
305
|
end
|
|
272
306
|
|
|
@@ -303,6 +337,7 @@ module Fp
|
|
|
303
337
|
fp list --gaps # the ones with no evidence yet — good places to start
|
|
304
338
|
fp show <slug> # details, acceptance criteria, and existing receipts
|
|
305
339
|
fp history <slug> # why it says what it says, and whether a change is pending
|
|
340
|
+
fp changes --source-url URL # requirement activity prompted by a work item
|
|
306
341
|
```
|
|
307
342
|
|
|
308
343
|
**The requirement catalog is the human-readable product specification, not a box to
|
|
@@ -342,6 +377,30 @@ module Fp
|
|
|
342
377
|
`fp propose-change` (or `--removal`) with the source and reason to put that decision before
|
|
343
378
|
a human. If the intent is unclear, stop and ask rather than making the catalog say both things.
|
|
344
379
|
|
|
380
|
+
**Treat existing tests as protected product evidence.** Before editing an existing test,
|
|
381
|
+
determine what expectation it already records and whether CI has passing evidence for it.
|
|
382
|
+
|
|
383
|
+
- If the test backs passing evidence, do not rewrite it to make the current implementation
|
|
384
|
+
or a new requirement fit. Explicitly flag the intended test edit in the work item and pull
|
|
385
|
+
request. If its expected behaviour would change, propose the requirement change and wait
|
|
386
|
+
for explicit human approval before editing the test; a newly green run must not hide that
|
|
387
|
+
the proof itself changed.
|
|
388
|
+
- When FeatureParity reports a changed bound test, run `fp changed-tests` (or add `--json`)
|
|
389
|
+
and compare the accepted and candidate source revisions with the requirement. Record the
|
|
390
|
+
exact candidate through `fp changed-tests assess <id> --candidate-fingerprint <sha256>`
|
|
391
|
+
with a plain-language explanation and one classification: `still_proves`,
|
|
392
|
+
`no_longer_proves`, or `uncertain`. The first resolves only that fingerprint; the others
|
|
393
|
+
remain open for a human product decision. A later fingerprint needs a new assessment.
|
|
394
|
+
- A new requirement does not retroactively own or supersede a pre-existing failing test —
|
|
395
|
+
whether the requirement is draft or active. Do not change that test, or adopt it as the new
|
|
396
|
+
requirement's proof, merely to make the new work pass. Surface the conflict and wait for
|
|
397
|
+
explicit human approval to change the existing expectation.
|
|
398
|
+
|
|
399
|
+
A changed-test finding is an audit signal alongside the testcase's independent CI outcome.
|
|
400
|
+
It is not a failed test and is separate from `suspect`, which means requirement wording
|
|
401
|
+
changed and its evidence needs re-verification. Neither permits rewriting a test before
|
|
402
|
+
product intent is decided.
|
|
403
|
+
|
|
345
404
|
If no requirement covers what you're building, propose one or a parent/child tree. Agents
|
|
346
405
|
always create **drafts**; a human activates them:
|
|
347
406
|
|
|
@@ -55,7 +55,11 @@ module Fp
|
|
|
55
55
|
|
|
56
56
|
result = client.propose_requirement_change(requirement['id'], params)
|
|
57
57
|
unless result[:ok]
|
|
58
|
-
|
|
58
|
+
if result[:status] == 409 && result[:details].is_a?(Hash)
|
|
59
|
+
print_conflict(result, slug: slug)
|
|
60
|
+
else
|
|
61
|
+
output.error(result[:error], status: result[:status])
|
|
62
|
+
end
|
|
59
63
|
exit 1
|
|
60
64
|
end
|
|
61
65
|
|
|
@@ -64,6 +68,47 @@ module Fp
|
|
|
64
68
|
|
|
65
69
|
private
|
|
66
70
|
|
|
71
|
+
# A 409 means the requirement already has an open proposal — the one-at-a-time rule
|
|
72
|
+
# (two "what it was" snapshots would make a rollback ambiguous). Rather than a dead
|
|
73
|
+
# end, show what the open proposal is and point at `fp comment` so the agent can
|
|
74
|
+
# raise its concern on the proposal a human is about to review.
|
|
75
|
+
def print_conflict(result, slug:)
|
|
76
|
+
pending = result[:details]['pending_change'] || {}
|
|
77
|
+
|
|
78
|
+
if output.json?
|
|
79
|
+
output.error(result[:error], status: result[:status])
|
|
80
|
+
return
|
|
81
|
+
end
|
|
82
|
+
|
|
83
|
+
warn "Error: #{result[:error]}"
|
|
84
|
+
warn ''
|
|
85
|
+
warn 'The open proposal:'
|
|
86
|
+
warn " Change: #{pending['id']}"
|
|
87
|
+
warn " Kind: #{pending['kind']}"
|
|
88
|
+
warn " Proposed: #{pending['actor']} (#{pending['actor_kind']})"
|
|
89
|
+
warn " Reason: #{pending['reason']}" if pending['reason']
|
|
90
|
+
print_conflict_source(pending['source'])
|
|
91
|
+
print_conflict_diff(pending['changes'])
|
|
92
|
+
warn ''
|
|
93
|
+
warn 'You cannot open a rival proposal, but you can comment on this one:'
|
|
94
|
+
warn " fp comment #{slug} --change #{pending['id']} --body \"...\" --project <project>"
|
|
95
|
+
warn 'Or wait for a human to review it. Run `fp history ' \
|
|
96
|
+
"#{slug} --project <project>` for the full diff."
|
|
97
|
+
end
|
|
98
|
+
|
|
99
|
+
def print_conflict_source(source)
|
|
100
|
+
return unless source.is_a?(Hash) && !source.empty?
|
|
101
|
+
|
|
102
|
+
warn " Source: #{source['type']}" if source['type']
|
|
103
|
+
warn " URL: #{source['url']}" if source['url']
|
|
104
|
+
end
|
|
105
|
+
|
|
106
|
+
def print_conflict_diff(changes)
|
|
107
|
+
return unless changes.is_a?(Hash) && !changes.empty?
|
|
108
|
+
|
|
109
|
+
warn ' Fields: ' + changes.keys.join(', ')
|
|
110
|
+
end
|
|
111
|
+
|
|
67
112
|
def validate_enums!(opts)
|
|
68
113
|
if opts[:category] && !VALID_CATEGORIES.include?(opts[:category])
|
|
69
114
|
output.error("Invalid category '#{opts[:category]}'. Must be one of: #{VALID_CATEGORIES.join(', ')}")
|