featureparity 0.0.27 → 0.0.29
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/fp/cli.rb +14 -2
- data/lib/fp/client.rb +79 -2
- data/lib/fp/commands/analyze.rb +17 -0
- data/lib/fp/commands/base.rb +16 -0
- data/lib/fp/commands/check.rb +38 -4
- data/lib/fp/commands/ci_report.rb +2 -2
- data/lib/fp/commands/comment.rb +1 -1
- data/lib/fp/commands/comments.rb +146 -0
- data/lib/fp/commands/coverage.rb +149 -0
- data/lib/fp/commands/doctor.rb +228 -0
- data/lib/fp/commands/help.rb +41 -1
- data/lib/fp/commands/history.rb +16 -1
- data/lib/fp/commands/init.rb +181 -117
- data/lib/fp/commands/list.rb +58 -4
- data/lib/fp/commands/mark.rb +167 -0
- data/lib/fp/commands/projects.rb +6 -1
- data/lib/fp/commands/propose.rb +435 -53
- data/lib/fp/commands/propose_change.rb +13 -7
- data/lib/fp/commands/question.rb +137 -0
- data/lib/fp/commands/report.rb +40 -4
- data/lib/fp/commands/run.rb +147 -0
- data/lib/fp/commands/show.rb +41 -3
- data/lib/fp/commands/spot_check.rb +458 -0
- data/lib/fp/commands/tutorial.rb +95 -0
- data/lib/fp/commands/whoami.rb +2 -0
- data/lib/fp/commands.rb +8 -0
- data/lib/fp/config.rb +12 -0
- data/lib/fp/junit.rb +74 -11
- data/lib/fp/repo_detector.rb +488 -0
- data/lib/fp/requirement_lint.rb +105 -0
- data/lib/fp/rspec_json.rb +54 -0
- data/lib/fp/tutorial/hexhaven/AGENTS.md +5 -0
- data/lib/fp/tutorial/hexhaven/README.md +29 -0
- data/lib/fp/tutorial/hexhaven/lib/hexhaven.rb +36 -0
- data/lib/fp/tutorial/hexhaven/test/junit_reporter.rb +134 -0
- data/lib/fp/tutorial/hexhaven/test/rules_test.rb +36 -0
- data/lib/fp/tutorial/hexhaven/test/test_helper.rb +6 -0
- data/lib/fp/version.rb +1 -1
- data/lib/fp.rb +1 -0
- data/skills/fp/SKILL.md +244 -105
- data/skills/fp/references/cli.md +304 -3
- metadata +19 -2
|
@@ -0,0 +1,458 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require 'open3'
|
|
4
|
+
|
|
5
|
+
module Fp
|
|
6
|
+
module Commands
|
|
7
|
+
# `fp spot-check [<slug>...]` — a random drug test for evidence (Fizzy #1787).
|
|
8
|
+
#
|
|
9
|
+
# A green receipt says a marked test ran and passed; it doesn't say the test checks
|
|
10
|
+
# what the requirement promises. This picks requirements that claim evidence — one at
|
|
11
|
+
# random by default, or N at random, every one, the ones named, or the ones bound to a
|
|
12
|
+
# given test — puts each acceptance next to the source of its locally marked tests,
|
|
13
|
+
# and hands the agent running it a brief: match each clause to an assertion, then
|
|
14
|
+
# break the product code and confirm the tests go red.
|
|
15
|
+
#
|
|
16
|
+
# It reads and prints only. It never writes evidence and never edits a test or a
|
|
17
|
+
# requirement; what the agent finds goes to a human.
|
|
18
|
+
#
|
|
19
|
+
# `--prompt` prints the same thing as instructions to hand an agent (`| claude -p`).
|
|
20
|
+
# `--jev` first asks Jev whether the tests appear to cover each acceptance, so the agent
|
|
21
|
+
# knows where to aim the probe; Jev is for paid plans, and a 402 just means the spot
|
|
22
|
+
# check carries on without its read.
|
|
23
|
+
class SpotCheck < Base
|
|
24
|
+
# Statuses that claim a test exists and proves the requirement. `stub`, `failing`
|
|
25
|
+
# and `suspect` already say something is wrong, so there's nothing to spot-check.
|
|
26
|
+
CLAIMED_STATUSES = %w[passing pr_passing present].freeze
|
|
27
|
+
|
|
28
|
+
# The excerpt stops here even if no end of the test was found.
|
|
29
|
+
MAX_EXCERPT_LINES = 120
|
|
30
|
+
# Generated or vendored trees never hold the repo's own tests.
|
|
31
|
+
SKIP_DIRS = %w[.git node_modules vendor tmp dist build coverage .bundle].freeze
|
|
32
|
+
MAX_FILE_BYTES = 1_000_000
|
|
33
|
+
|
|
34
|
+
BRIEF = [
|
|
35
|
+
'Read the acceptance and list each observable result and boundary it promises.',
|
|
36
|
+
'For each test, name the clause its assertions actually check. A clause that no ' \
|
|
37
|
+
'assertion checks is a gap, even when the test is green.',
|
|
38
|
+
'Mutation probe: make one small change to the product code (not the test) that ' \
|
|
39
|
+
'breaks the promise, run only these tests, and confirm at least one fails. Then ' \
|
|
40
|
+
'revert it and confirm `git status` is clean.',
|
|
41
|
+
"A test that stays green under the probe, or checks something the acceptance doesn't " \
|
|
42
|
+
"say, doesn't prove the requirement. Tell a human what you tried and what happened. " \
|
|
43
|
+
"Don't rewrite the test or the requirement as part of a spot check."
|
|
44
|
+
].freeze
|
|
45
|
+
|
|
46
|
+
help_summary 'Check that requirements\' tests prove what they promise (random, chosen, or all)'
|
|
47
|
+
help_usage 'fp spot-check [<slug>...] [--count <n> | --all | --test <file>[:<line>]] ' \
|
|
48
|
+
'[--surface <s>] [--seed <n>] [--path <dir>]'
|
|
49
|
+
help_flag :project, 'Project slug (falls back to .featureparity.yml)'
|
|
50
|
+
help_flag :count, 'Pick this many requirements at random instead of one'
|
|
51
|
+
help_flag :all, 'Check every active requirement that has evidence'
|
|
52
|
+
help_flag :test, 'Check the test(s) in this file, or the one at file:line (repeatable)'
|
|
53
|
+
help_flag :surface, 'Only pick from evidence on this surface, and only show its tests'
|
|
54
|
+
help_flag :seed, 'Random seed, to repeat a pick'
|
|
55
|
+
help_flag :path, 'Where to look for marked tests (default: the git checkout you are in)'
|
|
56
|
+
help_flag :prompt, 'Print the spot check as a prompt to hand an agent (e.g. pipe to claude -p)'
|
|
57
|
+
help_flag :jev, "Add Jev's first read of whether the tests cover each acceptance (paid plans)"
|
|
58
|
+
help_example 'fp spot-check', 'a random requirement with evidence'
|
|
59
|
+
help_example 'fp spot-check --count 5 --surface cli --json', 'five at random, for an agent on a schedule'
|
|
60
|
+
help_example 'fp spot-check print_container_qr print_label', 'chosen requirements'
|
|
61
|
+
help_example 'fp spot-check --all', 'every requirement with evidence'
|
|
62
|
+
help_example 'fp spot-check --test test/qr_test.rb:42', 'the test at that line, and its requirement'
|
|
63
|
+
help_example 'fp spot-check --count 3 --jev --prompt | claude -p', 'a scheduled run, end to end'
|
|
64
|
+
help_note 'Read-only: it records nothing and never edits a test or a requirement.'
|
|
65
|
+
|
|
66
|
+
KNOWN_FLAGS = {
|
|
67
|
+
project: :string, surface: :string, seed: :string, path: :string,
|
|
68
|
+
count: :string, all: :boolean, prompt: :boolean, jev: :boolean
|
|
69
|
+
}.freeze
|
|
70
|
+
|
|
71
|
+
# Jev reads at most this many tests per requirement (the server's cap).
|
|
72
|
+
JEV_MAX_TESTS = 10
|
|
73
|
+
|
|
74
|
+
def run(args)
|
|
75
|
+
test_refs, args = extract_repeated(args, '--test')
|
|
76
|
+
opts, slugs = parse_flags(args, KNOWN_FLAGS)
|
|
77
|
+
require_project!(opts)
|
|
78
|
+
mode = selection_mode(opts, slugs, test_refs)
|
|
79
|
+
if opts[:prompt] && output.json?
|
|
80
|
+
output.error('--prompt prints plain text for an agent; drop --json.')
|
|
81
|
+
exit 1
|
|
82
|
+
end
|
|
83
|
+
project_id = resolve_project_id(opts[:project])
|
|
84
|
+
|
|
85
|
+
seed = parse_seed(opts[:seed])
|
|
86
|
+
count = parse_count(opts[:count])
|
|
87
|
+
requirements = fetch_requirements(project_id)
|
|
88
|
+
evidence = fetch_evidence(project_id, opts[:surface])
|
|
89
|
+
root = File.expand_path(opts[:path] || repo_root(Dir.pwd))
|
|
90
|
+
tests_by_slug = find_tests(root, opts[:surface])
|
|
91
|
+
|
|
92
|
+
if mode == :test
|
|
93
|
+
tests_by_slug = select_tests(tests_by_slug, test_refs, root)
|
|
94
|
+
slugs = tests_by_slug.keys
|
|
95
|
+
end
|
|
96
|
+
|
|
97
|
+
chosen = choose(requirements, evidence, mode, slugs, count, Random.new(seed))
|
|
98
|
+
payload = {
|
|
99
|
+
'seed' => (seed if %i[random count].include?(mode)),
|
|
100
|
+
'search_root' => root,
|
|
101
|
+
'checks' => chosen.map do |requirement|
|
|
102
|
+
slug = requirement['slug']
|
|
103
|
+
{
|
|
104
|
+
'requirement' => requirement.slice('id', 'slug', 'title', 'why', 'acceptance', 'required_surfaces'),
|
|
105
|
+
'evidence' => evidence.select { |e| e['slug'] == slug },
|
|
106
|
+
'tests' => tests_by_slug.fetch(slug, [])
|
|
107
|
+
}
|
|
108
|
+
end,
|
|
109
|
+
'brief' => BRIEF
|
|
110
|
+
}.compact
|
|
111
|
+
payload['jev_note'] = add_jev_reads!(payload['checks']) if opts[:jev]
|
|
112
|
+
payload.compact!
|
|
113
|
+
|
|
114
|
+
if opts[:prompt]
|
|
115
|
+
print_prompt(payload)
|
|
116
|
+
else
|
|
117
|
+
output.success(payload) { print_report(payload) }
|
|
118
|
+
end
|
|
119
|
+
end
|
|
120
|
+
|
|
121
|
+
private
|
|
122
|
+
|
|
123
|
+
# One way of choosing per run, so it's never ambiguous what was checked.
|
|
124
|
+
def selection_mode(opts, slugs, test_refs)
|
|
125
|
+
modes = []
|
|
126
|
+
modes << :slugs unless slugs.empty?
|
|
127
|
+
modes << :all if opts[:all]
|
|
128
|
+
modes << :count if opts[:count]
|
|
129
|
+
modes << :test unless test_refs.empty?
|
|
130
|
+
if modes.length > 1
|
|
131
|
+
output.error('Choose requirements one way: slugs, --count, --all, or --test.')
|
|
132
|
+
exit 1
|
|
133
|
+
end
|
|
134
|
+
|
|
135
|
+
modes.first || :random
|
|
136
|
+
end
|
|
137
|
+
|
|
138
|
+
def parse_seed(value)
|
|
139
|
+
return Random.new_seed % 1_000_000 if value.nil?
|
|
140
|
+
|
|
141
|
+
unless value.to_s.match?(/\A\d+\z/)
|
|
142
|
+
output.error('--seed must be a whole number')
|
|
143
|
+
exit 1
|
|
144
|
+
end
|
|
145
|
+
value.to_i
|
|
146
|
+
end
|
|
147
|
+
|
|
148
|
+
def parse_count(value)
|
|
149
|
+
return 1 if value.nil?
|
|
150
|
+
|
|
151
|
+
unless value.to_s.match?(/\A[1-9]\d*\z/)
|
|
152
|
+
output.error('--count must be a whole number of at least 1')
|
|
153
|
+
exit 1
|
|
154
|
+
end
|
|
155
|
+
value.to_i
|
|
156
|
+
end
|
|
157
|
+
|
|
158
|
+
def fetch_requirements(project_id)
|
|
159
|
+
result = client.list_requirements(project_id: project_id)
|
|
160
|
+
fail_result(result) unless result[:ok]
|
|
161
|
+
result[:data]['requirements'] || []
|
|
162
|
+
end
|
|
163
|
+
|
|
164
|
+
# Every receipt the grid still believes. Rows CI has superseded are left out, so a
|
|
165
|
+
# stale `present` beside CI's verdict can't make a requirement look claimed.
|
|
166
|
+
def fetch_evidence(project_id, surface)
|
|
167
|
+
result = client.list_evidence(project_id, surface: surface)
|
|
168
|
+
fail_result(result) unless result[:ok]
|
|
169
|
+
|
|
170
|
+
superseded = Array(result[:data]['superseded_ids'])
|
|
171
|
+
(result[:data]['evidence'] || []).reject { |e| superseded.include?(e['id']) }
|
|
172
|
+
end
|
|
173
|
+
|
|
174
|
+
def choose(requirements, evidence, mode, slugs, count, rng)
|
|
175
|
+
return named(requirements, slugs) if %i[slugs test].include?(mode)
|
|
176
|
+
|
|
177
|
+
claimed = evidence.select { |e| CLAIMED_STATUSES.include?(e['status']) }.map { |e| e['slug'] }.uniq
|
|
178
|
+
# Sorted so a seed picks the same requirements however the server orders the list.
|
|
179
|
+
candidates = requirements.select { |r| r['status'] == 'active' && claimed.include?(r['slug']) }
|
|
180
|
+
.sort_by { |r| r['slug'] }
|
|
181
|
+
if candidates.empty?
|
|
182
|
+
output.error('No active requirement has evidence to spot-check yet.')
|
|
183
|
+
exit 1
|
|
184
|
+
end
|
|
185
|
+
return candidates if mode == :all
|
|
186
|
+
|
|
187
|
+
candidates.sample(count, random: rng).sort_by { |r| r['slug'] }
|
|
188
|
+
end
|
|
189
|
+
|
|
190
|
+
# Named requirements are checked whatever their status or evidence: asking for one
|
|
191
|
+
# is the point.
|
|
192
|
+
def named(requirements, slugs)
|
|
193
|
+
missing = slugs.reject { |slug| requirements.any? { |r| r['slug'] == slug } }
|
|
194
|
+
unless missing.empty?
|
|
195
|
+
output.error("Requirement(s) not found in project: #{missing.join(', ')}")
|
|
196
|
+
exit 1
|
|
197
|
+
end
|
|
198
|
+
|
|
199
|
+
slugs.uniq.map { |slug| requirements.find { |r| r['slug'] == slug } }
|
|
200
|
+
end
|
|
201
|
+
|
|
202
|
+
# Keep only the marked tests a --test reference names: every one in the file, or with
|
|
203
|
+
# a line, the one whose marker-to-end span holds it.
|
|
204
|
+
def select_tests(tests_by_slug, refs, root)
|
|
205
|
+
wanted = refs.map { |ref| parse_test_ref(ref, root) }
|
|
206
|
+
selected = tests_by_slug.transform_values do |tests|
|
|
207
|
+
tests.select do |t|
|
|
208
|
+
wanted.any? { |file, line| t['file'] == file && (line.nil? || (t['line']..t['end_line']).cover?(line)) }
|
|
209
|
+
end
|
|
210
|
+
end.reject { |_, tests| tests.empty? }
|
|
211
|
+
|
|
212
|
+
if selected.empty?
|
|
213
|
+
output.error("No fp: marked test found at #{refs.join(', ')} under #{root}.")
|
|
214
|
+
exit 1
|
|
215
|
+
end
|
|
216
|
+
selected
|
|
217
|
+
end
|
|
218
|
+
|
|
219
|
+
# `path[:line]`, with the path taken relative to the working directory or to root.
|
|
220
|
+
def parse_test_ref(ref, root)
|
|
221
|
+
path, line = ref.match(/\A(.+?)(?::(\d+))?\z/).captures
|
|
222
|
+
absolute = File.expand_path(path)
|
|
223
|
+
file = if absolute.start_with?("#{root}/")
|
|
224
|
+
absolute.delete_prefix("#{root}/")
|
|
225
|
+
else
|
|
226
|
+
path.delete_prefix('./')
|
|
227
|
+
end
|
|
228
|
+
[file, line&.to_i]
|
|
229
|
+
end
|
|
230
|
+
|
|
231
|
+
# Ask Jev about each check that has tests here, adding `jev` to it. Stops at the first
|
|
232
|
+
# refusal (unpaid, unconfigured, unreachable): the answer would be the same for the
|
|
233
|
+
# rest, and the spot check stands without it. Returns a note when it stopped.
|
|
234
|
+
def add_jev_reads!(checks)
|
|
235
|
+
checks.each do |check|
|
|
236
|
+
tests = check['tests'].first(JEV_MAX_TESTS)
|
|
237
|
+
.map { |t| t.slice('file', 'line', 'title', 'source') }
|
|
238
|
+
next if tests.empty?
|
|
239
|
+
|
|
240
|
+
result = client.review_spot_check_tests(check['requirement']['id'], tests)
|
|
241
|
+
return jev_refusal(result) unless result[:ok]
|
|
242
|
+
|
|
243
|
+
check['jev'] = result[:data]['review']
|
|
244
|
+
end
|
|
245
|
+
nil
|
|
246
|
+
end
|
|
247
|
+
|
|
248
|
+
def jev_refusal(result)
|
|
249
|
+
case result[:status]
|
|
250
|
+
when 402 then "Jev's read is for paid plans, so this spot check runs without it. (#{result[:error]})"
|
|
251
|
+
else "Jev couldn't read the tests, so this spot check runs without it: #{result[:error]}"
|
|
252
|
+
end
|
|
253
|
+
end
|
|
254
|
+
|
|
255
|
+
def repo_root(dir)
|
|
256
|
+
out, status = Open3.capture2('git', '-C', dir, 'rev-parse', '--show-toplevel', err: File::NULL)
|
|
257
|
+
status.success? ? out.strip : dir
|
|
258
|
+
rescue SystemCallError
|
|
259
|
+
dir
|
|
260
|
+
end
|
|
261
|
+
|
|
262
|
+
# Every fp:<slug> marker under root with the test it annotates, grouped by slug.
|
|
263
|
+
# One pass over the tree, however many requirements are being checked.
|
|
264
|
+
def find_tests(root, surface)
|
|
265
|
+
tests = Hash.new { |h, k| h[k] = [] }
|
|
266
|
+
source_files(root).each do |rel|
|
|
267
|
+
contents = read_text(File.join(root, rel))
|
|
268
|
+
next unless contents&.include?('fp:')
|
|
269
|
+
|
|
270
|
+
lines = contents.lines
|
|
271
|
+
JUnit.scan_markers(contents).each do |marker|
|
|
272
|
+
next if surface && !marker[:surfaces].empty? && !marker[:surfaces].include?(surface)
|
|
273
|
+
|
|
274
|
+
source, truncated = excerpt(lines, marker[:line] - 1)
|
|
275
|
+
tests[marker[:slug]] << {
|
|
276
|
+
'file' => rel,
|
|
277
|
+
'line' => marker[:line],
|
|
278
|
+
'end_line' => marker[:line] + source.lines.length - 1,
|
|
279
|
+
'title' => marker[:title],
|
|
280
|
+
'surfaces' => marker[:surfaces],
|
|
281
|
+
'test_type' => marker[:test_type],
|
|
282
|
+
'source' => source,
|
|
283
|
+
'truncated' => truncated
|
|
284
|
+
}
|
|
285
|
+
end
|
|
286
|
+
end
|
|
287
|
+
tests
|
|
288
|
+
end
|
|
289
|
+
|
|
290
|
+
# Tracked files when root is in a git checkout (so .gitignore is respected),
|
|
291
|
+
# otherwise a walk of the tree.
|
|
292
|
+
def source_files(root)
|
|
293
|
+
out, status = Open3.capture2('git', '-C', root, 'ls-files', '-z', err: File::NULL)
|
|
294
|
+
files = status.success? ? out.split("\0") : walk(root)
|
|
295
|
+
files.reject { |f| f.split('/').any? { |part| SKIP_DIRS.include?(part) } }.sort
|
|
296
|
+
rescue SystemCallError
|
|
297
|
+
walk(root).sort
|
|
298
|
+
end
|
|
299
|
+
|
|
300
|
+
def walk(root)
|
|
301
|
+
Dir.glob('**/*', base: root).select { |f| File.file?(File.join(root, f)) }
|
|
302
|
+
end
|
|
303
|
+
|
|
304
|
+
def read_text(path)
|
|
305
|
+
return nil unless File.file?(path) && File.size(path) <= MAX_FILE_BYTES
|
|
306
|
+
|
|
307
|
+
contents = File.read(path, mode: 'rb')
|
|
308
|
+
return nil if contents.include?("\0")
|
|
309
|
+
|
|
310
|
+
contents.force_encoding(Encoding::UTF_8).scrub
|
|
311
|
+
rescue SystemCallError
|
|
312
|
+
nil
|
|
313
|
+
end
|
|
314
|
+
|
|
315
|
+
# The marker line through the end of the test below it. The end is the first line
|
|
316
|
+
# back at the declaration's indent: kept when it closes the block (`end`, `}`, `})`),
|
|
317
|
+
# dropped when it starts the next thing (Python, or the next test).
|
|
318
|
+
def excerpt(lines, marker_idx)
|
|
319
|
+
decl_idx = declaration_index(lines, marker_idx)
|
|
320
|
+
return [lines[marker_idx, 10].join, false] unless decl_idx
|
|
321
|
+
|
|
322
|
+
indent = indent_of(lines[decl_idx])
|
|
323
|
+
last = decl_idx
|
|
324
|
+
truncated = true
|
|
325
|
+
((decl_idx + 1)...lines.length).each do |i|
|
|
326
|
+
break if i - marker_idx >= MAX_EXCERPT_LINES
|
|
327
|
+
|
|
328
|
+
line = lines[i]
|
|
329
|
+
if line.strip.empty? || indent_of(line) > indent
|
|
330
|
+
last = i
|
|
331
|
+
next
|
|
332
|
+
end
|
|
333
|
+
|
|
334
|
+
last = i if line.strip.match?(/\A(?:end\b|[}\])])/)
|
|
335
|
+
truncated = false
|
|
336
|
+
break
|
|
337
|
+
end
|
|
338
|
+
truncated = false if last == lines.length - 1
|
|
339
|
+
|
|
340
|
+
[lines[marker_idx..last].join.rstrip + "\n", truncated]
|
|
341
|
+
end
|
|
342
|
+
|
|
343
|
+
# The test's declaration: the first line of code below the marker, past any further
|
|
344
|
+
# comments and annotations (`@Test`).
|
|
345
|
+
def declaration_index(lines, marker_idx)
|
|
346
|
+
((marker_idx + 1)...[marker_idx + 1 + JUnit::DECL_SEARCH_WINDOW, lines.length].min).find do |i|
|
|
347
|
+
line = lines[i]
|
|
348
|
+
!(line.strip.empty? || line.match?(JUnit::COMMENT_LINE_RE) || line.match?(/\A\s*@\w/))
|
|
349
|
+
end
|
|
350
|
+
end
|
|
351
|
+
|
|
352
|
+
def indent_of(line)
|
|
353
|
+
line[/\A[ \t]*/].length
|
|
354
|
+
end
|
|
355
|
+
|
|
356
|
+
def print_report(payload)
|
|
357
|
+
checks = payload['checks']
|
|
358
|
+
if payload['seed']
|
|
359
|
+
puts "Picked at random (seed #{payload['seed']}; repeat with --seed #{payload['seed']})"
|
|
360
|
+
end
|
|
361
|
+
puts "Spot-checking #{checks.length} requirements." if checks.length > 1
|
|
362
|
+
checks.each { |check| print_check(check, payload['search_root']) }
|
|
363
|
+
|
|
364
|
+
puts "Note: #{payload['jev_note']}" if payload['jev_note']
|
|
365
|
+
puts 'Your brief, for each requirement:'
|
|
366
|
+
payload['brief'].each_with_index { |step, i| puts " #{i + 1}. #{step}" }
|
|
367
|
+
end
|
|
368
|
+
|
|
369
|
+
# The spot check as instructions for an agent working in this checkout: who it is,
|
|
370
|
+
# what to do for each requirement, the dossier, and the shape of the report back.
|
|
371
|
+
def print_prompt(payload)
|
|
372
|
+
checks = payload['checks']
|
|
373
|
+
# On stderr, so a pipe into an agent carries only the prompt.
|
|
374
|
+
warn "Note: #{payload['jev_note']}" if payload['jev_note']
|
|
375
|
+
puts "You are spot-checking #{checks.length == 1 ? 'one requirement' : "#{checks.length} requirements"} " \
|
|
376
|
+
"in the checkout at #{payload['search_root']}: do the tests bound to it actually prove what it " \
|
|
377
|
+
'promises? A green test only shows that it ran and passed.'
|
|
378
|
+
puts
|
|
379
|
+
puts 'For each requirement below:'
|
|
380
|
+
payload['brief'].each_with_index { |step, i| puts "#{i + 1}. #{step}" }
|
|
381
|
+
puts
|
|
382
|
+
puts 'Run tests with whatever this repository uses. Change only product code, one probe at a time, ' \
|
|
383
|
+
'and revert each probe before the next. If a requirement has no tests here, say so and move on.'
|
|
384
|
+
puts "Jev's read, where given, is a hint about where to aim the probe, not a verdict." if checks.any? { |c| c['jev'] }
|
|
385
|
+
checks.each { |check| print_check(check, payload['search_root']) }
|
|
386
|
+
puts 'When you are done, report for each requirement:'
|
|
387
|
+
puts '- slug, and a verdict: proves, does_not_prove, or uncertain'
|
|
388
|
+
puts '- each acceptance clause with no assertion that checks it'
|
|
389
|
+
puts '- the probe you made (file and change), the tests you ran, and which failed'
|
|
390
|
+
puts '- confirmation that `git status` is clean'
|
|
391
|
+
end
|
|
392
|
+
|
|
393
|
+
def print_check(check, search_root)
|
|
394
|
+
req = check['requirement']
|
|
395
|
+
puts
|
|
396
|
+
puts "Spot check: #{req['slug']}"
|
|
397
|
+
puts " #{req['title']}"
|
|
398
|
+
puts
|
|
399
|
+
puts "Why: #{req['why']}" if req['why'].to_s.strip != ''
|
|
400
|
+
puts 'Acceptance:'
|
|
401
|
+
req['acceptance'].to_s.each_line { |l| puts " #{l.rstrip}" }
|
|
402
|
+
puts
|
|
403
|
+
|
|
404
|
+
puts 'Evidence on record:'
|
|
405
|
+
if check['evidence'].empty?
|
|
406
|
+
puts ' (none)'
|
|
407
|
+
else
|
|
408
|
+
check['evidence'].each do |e|
|
|
409
|
+
location = [e['source_file'], e['source_line']].compact.join(':')
|
|
410
|
+
puts " #{e['surface']} #{e['status']} #{location}".rstrip
|
|
411
|
+
end
|
|
412
|
+
end
|
|
413
|
+
puts
|
|
414
|
+
|
|
415
|
+
tests = check['tests']
|
|
416
|
+
if tests.empty?
|
|
417
|
+
puts "No fp:#{req['slug']} marker found under #{search_root}."
|
|
418
|
+
puts 'The tests may live in another repo: run this from that checkout or pass --path.'
|
|
419
|
+
puts 'If they should be here, the marker is gone, and that is the finding.'
|
|
420
|
+
else
|
|
421
|
+
puts "#{tests.length} marked test(s) under #{search_root}:"
|
|
422
|
+
tests.each do |t|
|
|
423
|
+
puts
|
|
424
|
+
puts "--- #{t['file']}:#{t['line']}#{" (#{t['title']})" if t['title']}"
|
|
425
|
+
puts t['source']
|
|
426
|
+
puts ' … (cut off; read the rest in the file)' if t['truncated']
|
|
427
|
+
end
|
|
428
|
+
end
|
|
429
|
+
puts
|
|
430
|
+
print_jev(check['jev']) if check['jev']
|
|
431
|
+
end
|
|
432
|
+
|
|
433
|
+
def print_jev(review)
|
|
434
|
+
covers = review['covers_acceptance'] || {}
|
|
435
|
+
puts "Jev's first read: #{review['verdict']}" \
|
|
436
|
+
"#{" (covers the acceptance: #{percent(covers['probability'])}, confidence #{percent(covers['confidence'])})" if covers['probability']}"
|
|
437
|
+
gap = review['primary_gap']
|
|
438
|
+
puts " Likeliest gap: #{gap['description']}" if gap
|
|
439
|
+
Array(review['tests']).each do |t|
|
|
440
|
+
read = t['checks_promise'] || {}
|
|
441
|
+
next unless read['probability']
|
|
442
|
+
|
|
443
|
+
puts " #{t['file']}:#{t['line']} checks the promise: #{percent(read['probability'])}"
|
|
444
|
+
end
|
|
445
|
+
puts
|
|
446
|
+
end
|
|
447
|
+
|
|
448
|
+
def percent(value)
|
|
449
|
+
"#{(value.to_f * 100).round}%"
|
|
450
|
+
end
|
|
451
|
+
|
|
452
|
+
def fail_result(result)
|
|
453
|
+
output.error(result[:error], status: result[:status])
|
|
454
|
+
exit 1
|
|
455
|
+
end
|
|
456
|
+
end
|
|
457
|
+
end
|
|
458
|
+
end
|
|
@@ -0,0 +1,95 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require 'fileutils'
|
|
4
|
+
|
|
5
|
+
module Fp
|
|
6
|
+
module Commands
|
|
7
|
+
# fp tutorial [--dir PATH]
|
|
8
|
+
#
|
|
9
|
+
# Deals the newcomer a practice table (Fizzy #1446): Hexhaven, the made-up board game
|
|
10
|
+
# their tutorial project tracks, as a tiny repository whose tests run on Ruby alone.
|
|
11
|
+
# Two rules are already bound by fp: markers; the tallest-tower test is not, and the
|
|
12
|
+
# web tutorial's "Deal yourself in" step asks the reader to mark it and run
|
|
13
|
+
# `fp report`, which turns that Server cell in their sample matrix from missing to
|
|
14
|
+
# present. The rule slugs must match lambda/lib/tutorial_project.rb.
|
|
15
|
+
#
|
|
16
|
+
# Makes no API calls, like `fp init`: the files are the same for everyone, and the
|
|
17
|
+
# project slug is the tutorial's fixed one.
|
|
18
|
+
class Tutorial < Base
|
|
19
|
+
help_summary 'Write Hexhaven, a sample repo to practise marking and reporting tests'
|
|
20
|
+
help_usage 'fp tutorial [--dir PATH] [--restart]'
|
|
21
|
+
help_flag :dir, 'Where to write it (defaults to ./hexhaven)'
|
|
22
|
+
help_flag :restart, 'Replace an existing Hexhaven checkout with a fresh one'
|
|
23
|
+
help_example 'fp tutorial'
|
|
24
|
+
help_example 'fp tutorial --dir ~/code/hexhaven'
|
|
25
|
+
help_example 'fp tutorial --restart', 'Start the practice repo over, markers and all'
|
|
26
|
+
help_note "Pairs with the Tutorial in the web app, which opens your sample project.\n" \
|
|
27
|
+
'Makes no API calls. Only --restart replaces a directory, and only a Hexhaven one.'
|
|
28
|
+
|
|
29
|
+
KNOWN_FLAGS = { dir: :string, restart: :boolean }.freeze
|
|
30
|
+
TEMPLATE_DIR = File.expand_path('../tutorial/hexhaven', __dir__)
|
|
31
|
+
PROJECT_SLUG = 'fp-tutorial'
|
|
32
|
+
SURFACE = 'server'
|
|
33
|
+
|
|
34
|
+
def run(args)
|
|
35
|
+
opts, = parse_flags(args, KNOWN_FLAGS)
|
|
36
|
+
target = File.expand_path(opts[:dir] || 'hexhaven')
|
|
37
|
+
|
|
38
|
+
if File.exist?(target) && !(File.directory?(target) && Dir.empty?(target))
|
|
39
|
+
unless hexhaven?(target)
|
|
40
|
+
output.error("#{target} already exists and isn't a Hexhaven checkout. Pick another place with --dir.")
|
|
41
|
+
exit 1
|
|
42
|
+
end
|
|
43
|
+
unless opts[:restart]
|
|
44
|
+
output.error("#{target} already has Hexhaven in it. Run `fp tutorial --restart` to start it over.")
|
|
45
|
+
exit 1
|
|
46
|
+
end
|
|
47
|
+
FileUtils.rm_rf(target)
|
|
48
|
+
end
|
|
49
|
+
|
|
50
|
+
written = write_repo(target)
|
|
51
|
+
|
|
52
|
+
output.success({ dir: target, files: written }) do
|
|
53
|
+
puts "Dealt Hexhaven into #{target}"
|
|
54
|
+
puts
|
|
55
|
+
puts ' cd ' + display_path(target)
|
|
56
|
+
puts ' ruby -Ilib test/rules_test.rb'
|
|
57
|
+
puts
|
|
58
|
+
puts 'Then follow README.md.'
|
|
59
|
+
end
|
|
60
|
+
end
|
|
61
|
+
|
|
62
|
+
private
|
|
63
|
+
|
|
64
|
+
def write_repo(target)
|
|
65
|
+
FileUtils.mkdir_p(target)
|
|
66
|
+
files = Dir.glob('**/*', base: TEMPLATE_DIR).reject { |f| File.directory?(File.join(TEMPLATE_DIR, f)) }
|
|
67
|
+
files.each do |relative|
|
|
68
|
+
dest = File.join(target, relative)
|
|
69
|
+
FileUtils.mkdir_p(File.dirname(dest))
|
|
70
|
+
FileUtils.cp(File.join(TEMPLATE_DIR, relative), dest)
|
|
71
|
+
end
|
|
72
|
+
# Dotfiles stay out of the gem's file list, so they're written here instead.
|
|
73
|
+
File.write(File.join(target, ProjectConfig::FILENAME),
|
|
74
|
+
ProjectConfig.render(project: PROJECT_SLUG, surface: SURFACE))
|
|
75
|
+
File.write(File.join(target, '.gitignore'), "junit.xml\n")
|
|
76
|
+
# The agent brief is the one `fp init` writes, appended to the template's short
|
|
77
|
+
# Hexhaven intro, so "ask your agent to bind the test" works and stays current.
|
|
78
|
+
Init.write_agent_docs!(target, PROJECT_SLUG)
|
|
79
|
+
(files + [ProjectConfig::FILENAME, '.gitignore']).uniq.sort
|
|
80
|
+
end
|
|
81
|
+
|
|
82
|
+
# Only a directory this command wrote may be replaced: one whose project config
|
|
83
|
+
# names the tutorial project.
|
|
84
|
+
def hexhaven?(target)
|
|
85
|
+
config = File.join(target, ProjectConfig::FILENAME)
|
|
86
|
+
File.file?(config) && ProjectConfig.parse(config)['project'] == PROJECT_SLUG
|
|
87
|
+
end
|
|
88
|
+
|
|
89
|
+
def display_path(target)
|
|
90
|
+
relative = target.delete_prefix("#{Dir.pwd}/")
|
|
91
|
+
relative == target ? target : relative
|
|
92
|
+
end
|
|
93
|
+
end
|
|
94
|
+
end
|
|
95
|
+
end
|
data/lib/fp/commands/whoami.rb
CHANGED
|
@@ -32,6 +32,8 @@ module Fp
|
|
|
32
32
|
puts
|
|
33
33
|
puts " ⚠ Profile is tagged for environment '#{r[:profile_environment]}' " \
|
|
34
34
|
"but the resolved API URL points at '#{r[:environment]}'."
|
|
35
|
+
puts " A key only works in the environment that minted it. Mint one in the " \
|
|
36
|
+
"'#{r[:environment]}' web app (Account → Agents & API keys) and use it there."
|
|
35
37
|
end
|
|
36
38
|
|
|
37
39
|
unless r[:authenticated]
|
data/lib/fp/commands.rb
CHANGED
|
@@ -9,15 +9,23 @@ require_relative 'commands/list'
|
|
|
9
9
|
require_relative 'commands/show'
|
|
10
10
|
require_relative 'commands/propose'
|
|
11
11
|
require_relative 'commands/propose_change'
|
|
12
|
+
require_relative 'commands/mark'
|
|
12
13
|
require_relative 'commands/comment'
|
|
14
|
+
require_relative 'commands/question'
|
|
15
|
+
require_relative 'commands/comments'
|
|
13
16
|
require_relative 'commands/history'
|
|
14
17
|
require_relative 'commands/analyze'
|
|
15
18
|
require_relative 'commands/changes'
|
|
16
19
|
require_relative 'commands/report'
|
|
17
20
|
require_relative 'commands/ci_report'
|
|
18
21
|
require_relative 'commands/changed_tests'
|
|
22
|
+
require_relative 'commands/spot_check'
|
|
19
23
|
require_relative 'commands/check'
|
|
24
|
+
require_relative 'commands/coverage'
|
|
20
25
|
require_relative 'commands/init'
|
|
26
|
+
require_relative 'commands/tutorial'
|
|
27
|
+
require_relative 'commands/doctor'
|
|
28
|
+
require_relative 'commands/run'
|
|
21
29
|
require_relative 'commands/matrix'
|
|
22
30
|
require_relative 'commands/export'
|
|
23
31
|
require_relative 'commands/profile'
|
data/lib/fp/config.rb
CHANGED
|
@@ -63,6 +63,18 @@ module Fp
|
|
|
63
63
|
self.class.environment_for(@api_url)
|
|
64
64
|
end
|
|
65
65
|
|
|
66
|
+
# Said when a call comes back empty and the active profile was made for another
|
|
67
|
+
# environment: its key belongs to that environment and reaches nothing here.
|
|
68
|
+
def environment_mismatch_hint
|
|
69
|
+
tagged = profile_environment
|
|
70
|
+
here = environment
|
|
71
|
+
return nil if tagged.nil? || here.nil? || tagged == here
|
|
72
|
+
|
|
73
|
+
"Your profile '#{@profile_name}' holds a key for '#{tagged}', but fp is talking to '#{here}'. " \
|
|
74
|
+
"A key only works in the environment that minted it: mint one in the '#{here}' web app " \
|
|
75
|
+
"(Account → Agents & API keys), or unset FP_API_URL to go back to '#{tagged}'."
|
|
76
|
+
end
|
|
77
|
+
|
|
66
78
|
# The environment explicitly recorded on the active profile, if any.
|
|
67
79
|
# Falls back to nil when unset (callers derive from api_url instead).
|
|
68
80
|
def profile_environment
|