featureparity 0.0.16 → 0.0.18
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/fp/cli.rb +3 -0
- data/lib/fp/client.rb +87 -18
- data/lib/fp/commands/analyze.rb +213 -0
- data/lib/fp/commands/attach.rb +100 -136
- data/lib/fp/commands/attachments.rb +56 -83
- data/lib/fp/commands/changed_tests.rb +126 -0
- data/lib/fp/commands/ci_report.rb +133 -28
- data/lib/fp/commands/comment.rb +99 -0
- data/lib/fp/commands/help.rb +65 -4
- data/lib/fp/commands/init.rb +52 -12
- data/lib/fp/commands/propose_change.rb +46 -1
- data/lib/fp/commands/report.rb +64 -7
- data/lib/fp/commands.rb +3 -0
- data/lib/fp/junit.rb +50 -5
- data/lib/fp/source_fingerprint.rb +193 -0
- data/lib/fp/version.rb +1 -1
- data/lib/fp.rb +1 -0
- data/skills/fp/SKILL.md +63 -0
- data/skills/fp/references/cli.md +92 -1
- metadata +6 -2
data/lib/fp/commands/report.rb
CHANGED
|
@@ -86,7 +86,7 @@ module Fp
|
|
|
86
86
|
params[:work_item_url] = opts[:work_item] if opts[:work_item]
|
|
87
87
|
params[:ci_url] = opts[:ci_url] if opts[:ci_url]
|
|
88
88
|
params[:title] = opts[:title] if opts[:title]
|
|
89
|
-
params[:line] = opts[:line] if opts[:line]
|
|
89
|
+
params[:line] = opts[:line].to_i if opts[:line]
|
|
90
90
|
|
|
91
91
|
# Try to report evidence
|
|
92
92
|
result = client.report_evidence(params)
|
|
@@ -115,9 +115,26 @@ module Fp
|
|
|
115
115
|
exit 1
|
|
116
116
|
end
|
|
117
117
|
|
|
118
|
+
# The server echoes back the stored record, whose status/source may differ
|
|
119
|
+
# from what we asked for: CI evidence is authoritative, so an agent's
|
|
120
|
+
# present/stub never downgrades a cell CI already verified. Report the truth
|
|
121
|
+
# the server kept, not the state we requested, so the caller knows whether
|
|
122
|
+
# their report actually landed or was superseded by CI.
|
|
123
|
+
stored = result[:data].is_a?(Hash) ? (result[:data]['evidence'] || {}) : {}
|
|
124
|
+
stored_status = stored['status']
|
|
125
|
+
stored_source = stored['source']
|
|
126
|
+
superseded = ci_superseded?(state, stored_status, stored_source)
|
|
127
|
+
|
|
118
128
|
output.success(result[:data]) do
|
|
119
|
-
|
|
120
|
-
|
|
129
|
+
if superseded
|
|
130
|
+
puts "CI already verified '#{slug}' on surface '#{opts[:surface]}' (status: #{stored_status})."
|
|
131
|
+
puts "Your '#{state}' report was recorded as metadata but did NOT change the status —"
|
|
132
|
+
puts 'CI evidence is authoritative and is never downgraded by an agent report.'
|
|
133
|
+
puts ' Stored: ' + "#{stored_status} (source: #{stored_source})"
|
|
134
|
+
else
|
|
135
|
+
puts "Evidence reported for '#{slug}' on surface '#{opts[:surface]}'."
|
|
136
|
+
puts " State: #{stored_status || state}"
|
|
137
|
+
end
|
|
121
138
|
puts " Type: #{opts[:test_type]}" if opts[:test_type]
|
|
122
139
|
puts " File: #{opts[:file]}#{opts[:line] ? ":#{opts[:line]}" : ''}"
|
|
123
140
|
if opts[:pr]
|
|
@@ -140,6 +157,20 @@ module Fp
|
|
|
140
157
|
|
|
141
158
|
private
|
|
142
159
|
|
|
160
|
+
# CI statuses an agent can never set. Mirrors Evidence::CI_STATUSES on the
|
|
161
|
+
# server — kept here so the CLI can recognise, from the echoed record, when its
|
|
162
|
+
# present/stub report was superseded by an authoritative CI verdict.
|
|
163
|
+
CI_STATUSES = %w[passing pr_passing failing].freeze
|
|
164
|
+
|
|
165
|
+
# True when the server kept a CI verdict instead of applying our report: the
|
|
166
|
+
# stored evidence is CI-sourced with a CI status, yet we asked for present/stub.
|
|
167
|
+
def ci_superseded?(requested_state, stored_status, stored_source)
|
|
168
|
+
return false if stored_status.nil?
|
|
169
|
+
return false unless VALID_STATES.include?(requested_state)
|
|
170
|
+
|
|
171
|
+
stored_source == 'ci' && CI_STATUSES.include?(stored_status) && stored_status != requested_state
|
|
172
|
+
end
|
|
173
|
+
|
|
143
174
|
# Batch-upload evidence from a JUnit XML report.
|
|
144
175
|
def run_junit(opts)
|
|
145
176
|
require_project!(opts)
|
|
@@ -244,16 +275,34 @@ module Fp
|
|
|
244
275
|
exit 1
|
|
245
276
|
end
|
|
246
277
|
|
|
247
|
-
reported << { binding: binding, state: state, recorded: true
|
|
278
|
+
reported << { binding: binding, state: state, recorded: true,
|
|
279
|
+
stored_status: stored_status_from(report_result),
|
|
280
|
+
stored_source: stored_source_from(report_result) }
|
|
248
281
|
end
|
|
249
282
|
|
|
250
283
|
emit_junit_summary(opts, reported, skipped_unknown, unmatched, api_unavailable)
|
|
251
284
|
end
|
|
252
285
|
|
|
286
|
+
# Pull the stored status/source out of the server's echoed evidence record so the
|
|
287
|
+
# batch summary can flag cells where CI superseded the agent's present/stub.
|
|
288
|
+
def stored_status_from(report_result)
|
|
289
|
+
data = report_result[:data]
|
|
290
|
+
return nil unless data.is_a?(Hash)
|
|
291
|
+
|
|
292
|
+
(data['evidence'] || {})['status']
|
|
293
|
+
end
|
|
294
|
+
|
|
295
|
+
def stored_source_from(report_result)
|
|
296
|
+
data = report_result[:data]
|
|
297
|
+
return nil unless data.is_a?(Hash)
|
|
298
|
+
|
|
299
|
+
(data['evidence'] || {})['source']
|
|
300
|
+
end
|
|
301
|
+
|
|
253
302
|
def emit_junit_summary(opts, reported, skipped_unknown, unmatched, api_unavailable)
|
|
254
303
|
data = {
|
|
255
304
|
repo: opts[:repo],
|
|
256
|
-
reported: reported.map { |r| { slug: r[:binding].slug, surface: r[:binding].surface, test_type: r[:binding].test_type, file: r[:binding].file, line: r[:binding].line, state: r[:state], recorded: r[:recorded] } },
|
|
305
|
+
reported: reported.map { |r| { slug: r[:binding].slug, surface: r[:binding].surface, test_type: r[:binding].test_type, file: r[:binding].file, line: r[:binding].line, state: r[:state], recorded: r[:recorded], stored_status: r[:stored_status], superseded_by_ci: ci_superseded?(r[:state], r[:stored_status], r[:stored_source]) } },
|
|
257
306
|
skipped_unknown: skipped_unknown.map { |b| { slug: b.slug, surface: b.surface, file: b.file } },
|
|
258
307
|
unmatched_testcases: unmatched.map { |tc| { name: tc.name, file: tc.file } }
|
|
259
308
|
}
|
|
@@ -261,10 +310,18 @@ module Fp
|
|
|
261
310
|
output.success(data) do
|
|
262
311
|
puts "Reported evidence from #{opts[:junit]}:"
|
|
263
312
|
reported.each do |r|
|
|
264
|
-
|
|
313
|
+
superseded = ci_superseded?(r[:state], r[:stored_status], r[:stored_source])
|
|
314
|
+
marker = if !r[:recorded]
|
|
315
|
+
'•'
|
|
316
|
+
elsif superseded
|
|
317
|
+
'⊘'
|
|
318
|
+
else
|
|
319
|
+
'✓'
|
|
320
|
+
end
|
|
265
321
|
loc = r[:binding].line ? "#{r[:binding].file}:#{r[:binding].line}" : r[:binding].file
|
|
266
322
|
type = r[:binding].test_type ? " [#{r[:binding].test_type}]" : ''
|
|
267
|
-
|
|
323
|
+
shown_state = superseded ? "#{r[:stored_status]}, kept CI — #{r[:state]} ignored" : r[:state]
|
|
324
|
+
puts " #{marker} #{r[:binding].slug} → #{r[:binding].surface}#{type} (#{shown_state}) #{loc}"
|
|
268
325
|
end
|
|
269
326
|
|
|
270
327
|
unless skipped_unknown.empty?
|
data/lib/fp/commands.rb
CHANGED
|
@@ -7,10 +7,13 @@ require_relative 'commands/list'
|
|
|
7
7
|
require_relative 'commands/show'
|
|
8
8
|
require_relative 'commands/propose'
|
|
9
9
|
require_relative 'commands/propose_change'
|
|
10
|
+
require_relative 'commands/comment'
|
|
10
11
|
require_relative 'commands/history'
|
|
12
|
+
require_relative 'commands/analyze'
|
|
11
13
|
require_relative 'commands/changes'
|
|
12
14
|
require_relative 'commands/report'
|
|
13
15
|
require_relative 'commands/ci_report'
|
|
16
|
+
require_relative 'commands/changed_tests'
|
|
14
17
|
require_relative 'commands/check'
|
|
15
18
|
require_relative 'commands/init'
|
|
16
19
|
require_relative 'commands/matrix'
|
data/lib/fp/junit.rb
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
3
|
require 'rexml/document'
|
|
4
|
+
require_relative 'source_fingerprint'
|
|
4
5
|
|
|
5
6
|
module Fp
|
|
6
7
|
# Parses JUnit XML reports and binds testcases to fp:<slug> markers found in
|
|
@@ -40,6 +41,15 @@ module Fp
|
|
|
40
41
|
CiResult = Struct.new(:slug, :surface, :test_type, :status, :file, :line, :title, :test_count,
|
|
41
42
|
:failure_count, keyword_init: true)
|
|
42
43
|
|
|
44
|
+
# One exact testcase source receipt. Unlike CiResult, these are never aggregated:
|
|
45
|
+
# two tests may back the same requirement cell and each can change independently.
|
|
46
|
+
TestRevision = Struct.new(
|
|
47
|
+
:slug, :surface, :test_type, :testcase_key, :testcase_name, :classname, :outcome,
|
|
48
|
+
:repo, :file, :marker_line, :start_line, :end_line, :commit_sha,
|
|
49
|
+
:fingerprint, :fingerprint_scope,
|
|
50
|
+
keyword_init: true
|
|
51
|
+
)
|
|
52
|
+
|
|
43
53
|
# A failing testcase that carries no fp:<slug> marker at all — a "loose end". Not a
|
|
44
54
|
# cell on the parity grid (nothing binds it to a requirement) and not an orphan
|
|
45
55
|
# marker (there is no marker); it's a red test nobody is tracking against a
|
|
@@ -58,7 +68,7 @@ module Fp
|
|
|
58
68
|
# directory the runner was invoked from (vitest reports `src/pages/Foo.test.jsx` from
|
|
59
69
|
# inside `frontend/`), so reporting it verbatim produces a link to a file that doesn't
|
|
60
70
|
# exist at that path in the repo.
|
|
61
|
-
Match = Struct.new(:testcase, :marker, :precise, :repo_file, keyword_init: true)
|
|
71
|
+
Match = Struct.new(:testcase, :marker, :precise, :repo_file, :source_path, keyword_init: true)
|
|
62
72
|
|
|
63
73
|
# Evidence discovered by binding a testcase to a marker.
|
|
64
74
|
# `line` is the 1-based line number of the test in its source file: the
|
|
@@ -234,9 +244,13 @@ module Fp
|
|
|
234
244
|
# A slug is `failing` if any test claiming it went red, `stub` if every test claiming
|
|
235
245
|
# it was skipped (a skipped test is not proof of anything), and `passing` otherwise.
|
|
236
246
|
#
|
|
237
|
-
# Returns { results: [CiResult, ...],
|
|
238
|
-
|
|
247
|
+
# Returns { results: [CiResult, ...], test_revisions: [TestRevision, ...],
|
|
248
|
+
# unmatched: [TestCase], missing_surface: [...] }.
|
|
249
|
+
def ci_results(testcases, base_dir: Dir.pwd, default_surface: nil, default_test_type: nil,
|
|
250
|
+
commit_sha: nil, repo: nil)
|
|
239
251
|
aggregates = {}
|
|
252
|
+
test_revisions = []
|
|
253
|
+
fingerprint_cache = {}
|
|
240
254
|
matched = {}
|
|
241
255
|
missing_surface = []
|
|
242
256
|
|
|
@@ -251,6 +265,8 @@ module Fp
|
|
|
251
265
|
# The marker's own type pin wins; otherwise the --test-type default (nil → the
|
|
252
266
|
# server files it as `unspecified`). One CI verdict per (slug, surface, type).
|
|
253
267
|
test_type = marker[:test_type] || default_test_type
|
|
268
|
+
fingerprint = fingerprint_cache[[match.source_path, marker[:line]]] ||=
|
|
269
|
+
SourceFingerprint.build(match.source_path, marker)
|
|
254
270
|
|
|
255
271
|
surfaces.each do |surface|
|
|
256
272
|
if surface.nil? || surface.empty?
|
|
@@ -276,6 +292,18 @@ module Fp
|
|
|
276
292
|
agg[:title] ||= title
|
|
277
293
|
agg[:failing_title] ||= title if tc.failed?
|
|
278
294
|
end
|
|
295
|
+
|
|
296
|
+
if fingerprint && commit_sha
|
|
297
|
+
outcome = if tc.stub? then 'stub' elsif tc.failed? then 'failing' else 'passing' end
|
|
298
|
+
test_revisions << TestRevision.new(
|
|
299
|
+
slug: marker[:slug], surface: surface, test_type: test_type,
|
|
300
|
+
testcase_key: testcase_key(match.repo_file, tc), testcase_name: tc.name,
|
|
301
|
+
classname: tc.classname, outcome: outcome, repo: repo, file: match.repo_file,
|
|
302
|
+
marker_line: marker[:line], start_line: fingerprint.start_line,
|
|
303
|
+
end_line: fingerprint.end_line, commit_sha: commit_sha,
|
|
304
|
+
fingerprint: fingerprint.fingerprint, fingerprint_scope: fingerprint.scope
|
|
305
|
+
)
|
|
306
|
+
end
|
|
279
307
|
end
|
|
280
308
|
end
|
|
281
309
|
|
|
@@ -293,7 +321,12 @@ module Fp
|
|
|
293
321
|
end
|
|
294
322
|
|
|
295
323
|
unmatched = testcases.reject { |tc| matched[tc.object_id] }
|
|
296
|
-
{
|
|
324
|
+
{
|
|
325
|
+
results: results,
|
|
326
|
+
test_revisions: test_revisions,
|
|
327
|
+
unmatched: unmatched,
|
|
328
|
+
missing_surface: missing_surface.uniq
|
|
329
|
+
}
|
|
297
330
|
end
|
|
298
331
|
|
|
299
332
|
# The loose ends in a set of unmatched testcases: the ones that *failed* but carry no
|
|
@@ -326,11 +359,23 @@ module Fp
|
|
|
326
359
|
marker, precise = marker_for(tc, cases, markers)
|
|
327
360
|
next unless marker
|
|
328
361
|
|
|
329
|
-
yield Match.new(
|
|
362
|
+
yield Match.new(
|
|
363
|
+
testcase: tc, marker: marker, precise: precise, repo_file: repo_file,
|
|
364
|
+
source_path: File.expand_path(file, base_dir)
|
|
365
|
+
)
|
|
330
366
|
end
|
|
331
367
|
end
|
|
332
368
|
end
|
|
333
369
|
|
|
370
|
+
# Stable across body edits and line movement. A rename or move is intentionally a
|
|
371
|
+
# new identity; marker syntax has no durable per-test id from which to infer that the
|
|
372
|
+
# renamed testcase is the old one.
|
|
373
|
+
def testcase_key(repo_file, testcase)
|
|
374
|
+
Digest::SHA256.hexdigest(
|
|
375
|
+
[repo_file, testcase.classname, testcase.name].map(&:to_s).join("\0")
|
|
376
|
+
)
|
|
377
|
+
end
|
|
378
|
+
|
|
334
379
|
# The repository root containing `dir`, or nil when `dir` isn't inside one. #
|
|
335
380
|
# Found by walking up for a `.git` entry rather than shelling out to git, so it also
|
|
336
381
|
# works in a worktree (where `.git` is a file) and costs nothing.
|
|
@@ -0,0 +1,193 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require 'digest'
|
|
4
|
+
|
|
5
|
+
module Fp
|
|
6
|
+
# Produces a receipt for the exact test source CI executed. Isolation is deliberately
|
|
7
|
+
# conservative: when a complete test body cannot be identified confidently, the whole
|
|
8
|
+
# file is hashed and the scope says so instead of pretending a noisy file hash is an
|
|
9
|
+
# isolated test receipt.
|
|
10
|
+
module SourceFingerprint
|
|
11
|
+
Result = Struct.new(:fingerprint, :scope, :start_line, :end_line, keyword_init: true)
|
|
12
|
+
|
|
13
|
+
BRACE_EXTENSIONS = %w[.js .jsx .mjs .cjs .ts .tsx .kt .kts .swift .java .go .cs .php .rs .scala].freeze
|
|
14
|
+
|
|
15
|
+
module_function
|
|
16
|
+
|
|
17
|
+
def build(path, marker)
|
|
18
|
+
contents = File.binread(path)
|
|
19
|
+
lines = contents.lines
|
|
20
|
+
range = isolated_range(lines, marker, File.extname(path).downcase)
|
|
21
|
+
|
|
22
|
+
if range
|
|
23
|
+
first, last = range
|
|
24
|
+
source = lines[first..last].join
|
|
25
|
+
Result.new(
|
|
26
|
+
fingerprint: Digest::SHA256.hexdigest(source),
|
|
27
|
+
scope: 'isolated_test',
|
|
28
|
+
start_line: first + 1,
|
|
29
|
+
end_line: last + 1
|
|
30
|
+
)
|
|
31
|
+
else
|
|
32
|
+
Result.new(
|
|
33
|
+
fingerprint: Digest::SHA256.hexdigest(contents),
|
|
34
|
+
scope: 'whole_file',
|
|
35
|
+
start_line: 1,
|
|
36
|
+
end_line: lines.length
|
|
37
|
+
)
|
|
38
|
+
end
|
|
39
|
+
rescue Errno::ENOENT, Errno::EACCES
|
|
40
|
+
nil
|
|
41
|
+
end
|
|
42
|
+
|
|
43
|
+
def isolated_range(lines, marker, extension)
|
|
44
|
+
return nil unless marker[:title_kind] == :test
|
|
45
|
+
|
|
46
|
+
start_index = declaration_index(lines, marker)
|
|
47
|
+
return nil if start_index.nil? || start_index.negative? || start_index >= lines.length
|
|
48
|
+
|
|
49
|
+
case extension
|
|
50
|
+
when '.rb'
|
|
51
|
+
ruby_range(lines, start_index)
|
|
52
|
+
when '.py'
|
|
53
|
+
python_range(lines, start_index)
|
|
54
|
+
when *BRACE_EXTENSIONS
|
|
55
|
+
brace_range(lines, start_index)
|
|
56
|
+
end
|
|
57
|
+
end
|
|
58
|
+
|
|
59
|
+
def declaration_index(lines, marker)
|
|
60
|
+
marker_index = marker[:line].to_i - 1
|
|
61
|
+
((marker_index + 1)...[marker_index + 41, lines.length].min).each do |index|
|
|
62
|
+
line = lines[index]
|
|
63
|
+
stripped = line.strip
|
|
64
|
+
next if stripped.empty? || stripped.match?(%r{\A(?:#|//|--|/\*|\*)})
|
|
65
|
+
|
|
66
|
+
return index
|
|
67
|
+
end
|
|
68
|
+
nil
|
|
69
|
+
end
|
|
70
|
+
|
|
71
|
+
def ruby_range(lines, start_index)
|
|
72
|
+
declaration = strip_ruby_literals(lines[start_index])
|
|
73
|
+
return nil if declaration.nil?
|
|
74
|
+
return nil unless declaration.match?(/\bdef\s+test_?\w+/i) ||
|
|
75
|
+
declaration.match?(/\b(?:it|specify|scenario)\b.*\bdo\b/)
|
|
76
|
+
|
|
77
|
+
depth = 0
|
|
78
|
+
(start_index...lines.length).each do |index|
|
|
79
|
+
code = strip_ruby_literals(lines[index])
|
|
80
|
+
return nil if code.nil?
|
|
81
|
+
|
|
82
|
+
depth += code.scan(/\bdo\b/).length
|
|
83
|
+
depth += 1 if code.match?(/^\s*(?:def|class|module|if|unless|case|begin|while|until|for)\b/)
|
|
84
|
+
depth -= code.scan(/\bend\b/).length
|
|
85
|
+
return [start_index, index] if depth.zero? && index >= start_index
|
|
86
|
+
return nil if depth.negative?
|
|
87
|
+
end
|
|
88
|
+
nil
|
|
89
|
+
end
|
|
90
|
+
|
|
91
|
+
def strip_ruby_literals(line)
|
|
92
|
+
return nil if line.match?(/<<[-~]?['"`]?\w+/)
|
|
93
|
+
|
|
94
|
+
line.gsub(/'(?:\\.|[^'\\])*'|"(?:\\.|[^"\\])*"/, '').sub(/#.*/, '')
|
|
95
|
+
end
|
|
96
|
+
|
|
97
|
+
def python_range(lines, start_index)
|
|
98
|
+
declaration = lines[start_index]
|
|
99
|
+
return nil unless declaration.match?(/^\s*(?:async\s+)?def\s+test_?\w+.*:\s*(?:#.*)?$/i)
|
|
100
|
+
|
|
101
|
+
indent = declaration[/\A\s*/].length
|
|
102
|
+
last_body = nil
|
|
103
|
+
((start_index + 1)...lines.length).each do |index|
|
|
104
|
+
line = lines[index]
|
|
105
|
+
next if line.strip.empty? || line.lstrip.start_with?('#')
|
|
106
|
+
|
|
107
|
+
current_indent = line[/\A\s*/].length
|
|
108
|
+
break if current_indent <= indent
|
|
109
|
+
|
|
110
|
+
last_body = index
|
|
111
|
+
end
|
|
112
|
+
last_body ? [start_index, last_body] : nil
|
|
113
|
+
end
|
|
114
|
+
|
|
115
|
+
def brace_range(lines, start_index)
|
|
116
|
+
depth = 0
|
|
117
|
+
opened = false
|
|
118
|
+
in_block_comment = false
|
|
119
|
+
|
|
120
|
+
(start_index...lines.length).each do |index|
|
|
121
|
+
sanitized, in_block_comment = strip_brace_literals(lines[index], in_block_comment)
|
|
122
|
+
return nil if sanitized.nil?
|
|
123
|
+
|
|
124
|
+
sanitized.each_char do |char|
|
|
125
|
+
if char == '{'
|
|
126
|
+
depth += 1
|
|
127
|
+
opened = true
|
|
128
|
+
elsif char == '}' && opened
|
|
129
|
+
depth -= 1
|
|
130
|
+
return nil if depth.negative?
|
|
131
|
+
end
|
|
132
|
+
end
|
|
133
|
+
|
|
134
|
+
return [start_index, index] if opened && depth.zero?
|
|
135
|
+
return nil if !opened && index - start_index >= 10
|
|
136
|
+
end
|
|
137
|
+
nil
|
|
138
|
+
end
|
|
139
|
+
|
|
140
|
+
# Remove comments and quoted strings before balancing braces. Template literals are
|
|
141
|
+
# intentionally unsupported because interpolation can contain real code braces; those
|
|
142
|
+
# files use the labelled whole-file fallback rather than risk a false boundary.
|
|
143
|
+
def strip_brace_literals(line, in_block_comment)
|
|
144
|
+
return [nil, in_block_comment] if line.include?('`')
|
|
145
|
+
|
|
146
|
+
output = +''
|
|
147
|
+
quote = nil
|
|
148
|
+
escaped = false
|
|
149
|
+
index = 0
|
|
150
|
+
while index < line.length
|
|
151
|
+
char = line[index]
|
|
152
|
+
pair = line[index, 2]
|
|
153
|
+
|
|
154
|
+
if in_block_comment
|
|
155
|
+
if pair == '*/'
|
|
156
|
+
in_block_comment = false
|
|
157
|
+
index += 2
|
|
158
|
+
else
|
|
159
|
+
index += 1
|
|
160
|
+
end
|
|
161
|
+
next
|
|
162
|
+
end
|
|
163
|
+
|
|
164
|
+
if quote
|
|
165
|
+
if escaped
|
|
166
|
+
escaped = false
|
|
167
|
+
elsif char == '\\'
|
|
168
|
+
escaped = true
|
|
169
|
+
elsif char == quote
|
|
170
|
+
quote = nil
|
|
171
|
+
end
|
|
172
|
+
index += 1
|
|
173
|
+
next
|
|
174
|
+
end
|
|
175
|
+
|
|
176
|
+
break if pair == '//'
|
|
177
|
+
if pair == '/*'
|
|
178
|
+
in_block_comment = true
|
|
179
|
+
index += 2
|
|
180
|
+
next
|
|
181
|
+
end
|
|
182
|
+
if char == '"' || char == "'"
|
|
183
|
+
quote = char
|
|
184
|
+
else
|
|
185
|
+
output << char
|
|
186
|
+
end
|
|
187
|
+
index += 1
|
|
188
|
+
end
|
|
189
|
+
|
|
190
|
+
[output, in_block_comment]
|
|
191
|
+
end
|
|
192
|
+
end
|
|
193
|
+
end
|
data/lib/fp/version.rb
CHANGED
data/lib/fp.rb
CHANGED
data/skills/fp/SKILL.md
CHANGED
|
@@ -86,6 +86,27 @@ requirement around it. Identify which promise matches product intent, then use
|
|
|
86
86
|
`fp propose-change` (or `--removal`) with the source and reason to put that decision before a
|
|
87
87
|
human. If the intent is unclear, stop and ask rather than making the catalog say both things.
|
|
88
88
|
|
|
89
|
+
**Treat existing tests as protected product evidence.** Before editing an existing test,
|
|
90
|
+
determine what expectation it already records and whether CI has passing evidence for it.
|
|
91
|
+
|
|
92
|
+
- If the test backs passing evidence, do not rewrite it to make the current implementation or a
|
|
93
|
+
new requirement fit. Explicitly flag the intended test edit in the work item and pull request.
|
|
94
|
+
If its expected behaviour would change, propose the requirement change and wait for explicit
|
|
95
|
+
human approval before editing the test; a newly green run must not hide that the proof changed.
|
|
96
|
+
- When FeatureParity reports a changed bound test, run `fp changed-tests` (or add `--json`) and
|
|
97
|
+
compare its accepted and candidate revisions with the requirement. Record a plain-language
|
|
98
|
+
assessment for the exact candidate fingerprint with `fp changed-tests assess`: `still_proves`
|
|
99
|
+
resolves only that fingerprint; `no_longer_proves` and `uncertain` remain open for a human
|
|
100
|
+
product decision. A later fingerprint always needs a new assessment.
|
|
101
|
+
- A new requirement does not retroactively own or supersede a pre-existing failing test, whether
|
|
102
|
+
it is draft or active. Do not change that test, or adopt it as the new requirement's proof,
|
|
103
|
+
merely to make new work pass. Surface the conflict and wait for explicit human approval to
|
|
104
|
+
change the existing expectation.
|
|
105
|
+
|
|
106
|
+
A changed-test finding is an audit signal alongside the independent CI outcome, not a failed
|
|
107
|
+
test. It is also separate from `suspect`, which means requirement wording changed and its
|
|
108
|
+
evidence needs re-verification. Neither permits rewriting a test before product intent is decided.
|
|
109
|
+
|
|
89
110
|
### 2. Add the fp:<slug> marker to tests
|
|
90
111
|
|
|
91
112
|
Place a comment immediately before the test to bind it to a requirement:
|
|
@@ -118,6 +139,16 @@ fp report <slug> \
|
|
|
118
139
|
|
|
119
140
|
**Agents report `present` (default) or `stub` only. Never `passing` or `failing`.**
|
|
120
141
|
|
|
142
|
+
**Don't re-report a cell CI already covers.** CI evidence (`passing`, `pr_passing`,
|
|
143
|
+
`failing`) is authoritative. If CI has already run the test for a surface, a `present`
|
|
144
|
+
report adds nothing — it won't upgrade the cell, and the server will keep the CI verdict
|
|
145
|
+
rather than downgrade it to a bare marker. Check first with `fp show <slug> --project X`
|
|
146
|
+
(or `fp matrix`): if the surface already shows a CI status, skip the report. Report
|
|
147
|
+
`present`/`stub` for surfaces CI hasn't reached yet — e.g. a brand-new test before its
|
|
148
|
+
first CI run, or local work where CI isn't wired up. When you do report over a CI-covered
|
|
149
|
+
cell anyway, `fp report` now tells you it was superseded (the status stays CI's, your
|
|
150
|
+
report lands as metadata only) so you can see the no-op instead of assuming it took.
|
|
151
|
+
|
|
121
152
|
### 4. Report evidence from JUnit XML (batch upload without CI)
|
|
122
153
|
|
|
123
154
|
When CI isn't available, run the suite locally and upload the report:
|
|
@@ -319,6 +350,37 @@ FeatureParity section in `AGENTS.md`):
|
|
|
319
350
|
fp init --project <slug> --surface <surface>
|
|
320
351
|
```
|
|
321
352
|
|
|
353
|
+
### Triage requirement quality (`fp analyze`)
|
|
354
|
+
|
|
355
|
+
`fp analyze` checks how well requirements are written and whether they belong in the
|
|
356
|
+
catalog at all. It is **read-only: it never proposes anything.** Sweep first, report, and
|
|
357
|
+
only file the proposals a human picks — every proposal moves its requirement back to draft
|
|
358
|
+
and marks its evidence `suspect`, and a requirement holds one open proposal at a time, so
|
|
359
|
+
proposing on every flag could put dozens of requirements in review at once.
|
|
360
|
+
|
|
361
|
+
```bash
|
|
362
|
+
fp analyze --project stowzilla # sweep active + draft requirements
|
|
363
|
+
fp analyze print_container_qr --project stowzilla # one requirement, with suggestions
|
|
364
|
+
fp analyze --project stowzilla --conflicts # also check for contradictions (slower)
|
|
365
|
+
fp analyze --project stowzilla --cached # read saved results only, no model calls
|
|
366
|
+
fp analyze --project stowzilla --json
|
|
367
|
+
```
|
|
368
|
+
|
|
369
|
+
It goes one requirement at a time. Results are cached until the wording changes, so a
|
|
370
|
+
re-run only pays for requirements that are new or edited. Requirements with a change
|
|
371
|
+
already awaiting review are skipped.
|
|
372
|
+
|
|
373
|
+
Each finding suggests a kind of follow-up. Treat it as a signal that something is wrong,
|
|
374
|
+
not as the fix — you still write the proposal.
|
|
375
|
+
|
|
376
|
+
| Finding | Suggested follow-up |
|
|
377
|
+
|---------|---------------------|
|
|
378
|
+
| Possible conflict with another requirement | Stop and ask a human — a contradiction is a product decision. Read both requirements first; the analyzer says *that* they clash, not *why* |
|
|
379
|
+
| Duplicate (the report names `superseded by: <slug>`) | `fp propose-change <slug> --removal --reason "superseded by <other>"` |
|
|
380
|
+
| Fragment of a broader promise (`is_focused_promise` fails) | Fold it into the parent's acceptance: `fp propose-change <parent>` for the wording, then `--removal` for the fragment |
|
|
381
|
+
| Not a product promise / not observable | A human decides whether it belongs; don't guess |
|
|
382
|
+
| Missing why, vague acceptance, implementation detail, boilerplate | `fp propose-change <slug> --why "..." --acceptance "..."` |
|
|
383
|
+
|
|
322
384
|
## Quick reference
|
|
323
385
|
|
|
324
386
|
| Task | Command |
|
|
@@ -333,6 +395,7 @@ fp init --project <slug> --surface <surface>
|
|
|
333
395
|
| Propose a change | `fp propose-change <slug> --project X --why "..." --reason "..."` |
|
|
334
396
|
| Propose a deletion | `fp propose-change <slug> --project X --removal --reason "..."` |
|
|
335
397
|
| View change history | `fp history <slug> --project X` |
|
|
398
|
+
| Triage requirement quality | `fp analyze --project X` |
|
|
336
399
|
| View work-item activity | `fp changes --project X --source-url URL` |
|
|
337
400
|
| Enforce parity | `fp check --project X` |
|
|
338
401
|
| Gate on this run | `fp check --project X --surface Y --junit file.xml` |
|
data/skills/fp/references/cli.md
CHANGED
|
@@ -146,6 +146,12 @@ an existing promise is a contradiction signal: decide which promise reflects pro
|
|
|
146
146
|
then record that decision with `fp propose-change` or `--removal`. Do not weaken the test or add
|
|
147
147
|
a second requirement around the conflict; ask a human when product intent is unclear.
|
|
148
148
|
|
|
149
|
+
Existing tests are protected product evidence. Flag any intended edit to a test that backs
|
|
150
|
+
passing evidence; if its expected behaviour would change, wait for explicit human approval of
|
|
151
|
+
the requirement change before editing it. A new requirement does not supersede a pre-existing
|
|
152
|
+
failing test merely because the requirement was drafted or activated. Surface that conflict and
|
|
153
|
+
do not change or adopt the old test as proof without explicit human approval.
|
|
154
|
+
|
|
149
155
|
```bash
|
|
150
156
|
fp propose --project stowzilla \
|
|
151
157
|
--slug current_shipment_status \
|
|
@@ -270,6 +276,37 @@ the review decision where one was made. An open proposal is called out at the en
|
|
|
270
276
|
|
|
271
277
|
`--project` is optional when the repo has a `.featureparity.yml`.
|
|
272
278
|
|
|
279
|
+
### fp analyze
|
|
280
|
+
|
|
281
|
+
Triage requirement quality: duplicates, fragments of a broader promise, vague wording,
|
|
282
|
+
possible conflicts. **Read-only — it never files a proposal.** Report first; a human
|
|
283
|
+
picks which proposals to file (each moves its requirement to draft).
|
|
284
|
+
|
|
285
|
+
```bash
|
|
286
|
+
fp analyze --project stowzilla # active + draft requirements
|
|
287
|
+
fp analyze print_container_qr --project stowzilla # one requirement, with suggestions
|
|
288
|
+
fp analyze --project stowzilla --status active # narrow the sweep
|
|
289
|
+
fp analyze --project stowzilla --conflicts # also check contradictions (slower)
|
|
290
|
+
fp analyze --project stowzilla --force # re-run even if the result is current
|
|
291
|
+
fp analyze --project stowzilla --cached # saved results only, no model calls
|
|
292
|
+
fp analyze --project stowzilla --json
|
|
293
|
+
```
|
|
294
|
+
|
|
295
|
+
Analysis costs one model call per requirement, so the sweep runs one request at a time
|
|
296
|
+
rather than asking the server to do the whole catalog at once (which cannot finish inside
|
|
297
|
+
the API timeout). Results are cached until a requirement's title, why, or acceptance
|
|
298
|
+
changes, so re-running only pays for new or edited requirements. A requirement with a
|
|
299
|
+
change awaiting review is skipped.
|
|
300
|
+
|
|
301
|
+
The report groups findings by suggested follow-up: **conflict** (ask a human),
|
|
302
|
+
**propose removal** (duplicate — with `superseded by: <slug>` — or a fragment of a broader
|
|
303
|
+
promise), **needs human** (not clearly a product promise), **propose change** (wording
|
|
304
|
+
issues), plus in-review, not-analyzed (`--cached` only), and error. `--conflicts` adds
|
|
305
|
+
the other requirement's slug, a probability, and both acceptance clauses side by side —
|
|
306
|
+
the analyzer flags the clash but cannot explain it.
|
|
307
|
+
|
|
308
|
+
`--project` is optional when the repo has a `.featureparity.yml`.
|
|
309
|
+
|
|
273
310
|
### fp changes
|
|
274
311
|
|
|
275
312
|
Show requirement activity prompted by a work item, rather than opening every requirement's
|
|
@@ -295,6 +332,60 @@ item says FeatureParity cannot determine the activity; a linked work item with n
|
|
|
295
332
|
explicitly says no requirement changes were recorded. Generated `fp init` workflows configure
|
|
296
333
|
both outputs before the parity gate.
|
|
297
334
|
|
|
335
|
+
### fp changed-tests
|
|
336
|
+
|
|
337
|
+
List every unresolved audit finding for a bound test whose source fingerprint differs from
|
|
338
|
+
the accepted revision. The queue is independent of the testcase's passing/failing/stub outcome
|
|
339
|
+
and from `suspect` evidence caused by changed requirement wording.
|
|
340
|
+
|
|
341
|
+
```bash
|
|
342
|
+
fp changed-tests # project from .featureparity.yml
|
|
343
|
+
fp changed-tests --project stowzilla --surface api
|
|
344
|
+
fp changed-tests --json # complete machine-readable queue
|
|
345
|
+
```
|
|
346
|
+
|
|
347
|
+
Each finding includes the current requirement and acceptance rule, surface and optional test
|
|
348
|
+
type, testcase and CI outcome, isolated-test or labelled whole-file fingerprint scope, and the
|
|
349
|
+
accepted and candidate fingerprints, commits, source locations, and links.
|
|
350
|
+
|
|
351
|
+
After comparing those exact revisions with the requirement, an authorized agent records a
|
|
352
|
+
plain-language assessment:
|
|
353
|
+
|
|
354
|
+
```bash
|
|
355
|
+
fp changed-tests assess <finding-id> \
|
|
356
|
+
--candidate-fingerprint <sha256> \
|
|
357
|
+
--classification still_proves \
|
|
358
|
+
--explanation "The refactor changes setup only; the same observable result is asserted."
|
|
359
|
+
```
|
|
360
|
+
|
|
361
|
+
Classifications are `still_proves`, `no_longer_proves`, and `uncertain`. `still_proves`
|
|
362
|
+
resolves only the supplied candidate fingerprint; if it came from a PR, that exact fingerprint
|
|
363
|
+
becomes the accepted baseline only after a default-branch CI report observes it. The other two
|
|
364
|
+
classifications remain open for a human product decision. A stale candidate is rejected, and a
|
|
365
|
+
later fingerprint always requires a new assessment. Assessing never changes the requirement
|
|
366
|
+
wording or CI outcome.
|
|
367
|
+
|
|
368
|
+
#### Upgrading an existing project
|
|
369
|
+
|
|
370
|
+
Changed-test auditing requires fp 0.0.17 or newer and the corresponding backend deployment.
|
|
371
|
+
Hosted-project users do not migrate project data or configuration.
|
|
372
|
+
|
|
373
|
+
```bash
|
|
374
|
+
gem update featureparity
|
|
375
|
+
fp version # 0.0.17 or newer
|
|
376
|
+
```
|
|
377
|
+
|
|
378
|
+
Fresh CI jobs that run `gem install featureparity` automatically pick up the release; update
|
|
379
|
+
any Bundler or install-command pin explicitly. Keep the existing `fp ci-report` call and run it
|
|
380
|
+
once on the default branch: the upgraded CLI sends source receipts automatically, and that run
|
|
381
|
+
lazily establishes accepted baselines without a backfill. Report trusted pull requests with
|
|
382
|
+
`--pr <url>` to open findings before merge while preserving the default-branch baseline.
|
|
383
|
+
|
|
384
|
+
The workflow recipe generated by fp 0.0.17 shows both paths. Adapt an existing customized
|
|
385
|
+
workflow manually. `fp init --force` overwrites `.featureparity.yml` and the generated workflow,
|
|
386
|
+
so reserve it for unmodified generated files, preserve the original project/surface options,
|
|
387
|
+
and review the resulting diff.
|
|
388
|
+
|
|
298
389
|
### fp matrix
|
|
299
390
|
|
|
300
391
|
Display the parity matrix. Cells show the state of the evidence filed for each
|
|
@@ -398,7 +489,7 @@ fp init --project stowzilla --dir path/to/repo
|
|
|
398
489
|
| File | Purpose |
|
|
399
490
|
|------|---------|
|
|
400
491
|
| `.featureparity.yml` | declares the project slug, so `--project` becomes optional |
|
|
401
|
-
| `.github/workflows/feature-parity.yml` | the CI gate
|
|
492
|
+
| `.github/workflows/feature-parity.yml` | the CI gate, PR requirement-change summary, and adaptable PR/default-branch reporting recipe |
|
|
402
493
|
| `AGENTS.md` (appended) | instructions telling agents to mark and report their tests |
|
|
403
494
|
|
|
404
495
|
Existing files are skipped unless `--force`. The `AGENTS.md` section is appended between
|