scully 0.0.1 → 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +33 -0
- data/README.md +272 -9
- data/Rakefile +5 -1
- data/exe/scully +6 -0
- data/lib/scully/analyzer.rb +90 -0
- data/lib/scully/cli.rb +126 -0
- data/lib/scully/discovery.rb +53 -0
- data/lib/scully/frameworks/detector.rb +86 -0
- data/lib/scully/frameworks/helpers.rb +215 -0
- data/lib/scully/frameworks/minitest.rb +305 -0
- data/lib/scully/frameworks/rspec.rb +386 -0
- data/lib/scully/model.rb +48 -0
- data/lib/scully/render/report.rb +79 -0
- data/lib/scully/render/snippet.rb +92 -0
- data/lib/scully/render/terminal.rb +97 -0
- data/lib/scully/render/why.rb +146 -0
- data/lib/scully/rule.rb +85 -0
- data/lib/scully/rules/no_assertion.rb +51 -0
- data/lib/scully/rules/parse.rb +34 -0
- data/lib/scully/rules/tautology.rb +101 -0
- data/lib/scully/source_file.rb +53 -0
- data/lib/scully/version.rb +1 -1
- data/lib/scully.rb +18 -1
- metadata +39 -6
- data/sig/test_scully.rbs +0 -3
|
@@ -0,0 +1,97 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Scully
|
|
4
|
+
module Render
|
|
5
|
+
# Decides whether output may use colour and OSC 8 hyperlinks, and applies them.
|
|
6
|
+
#
|
|
7
|
+
# Neither can be queried from the terminal, so both are guessed from the environment.
|
|
8
|
+
# Colour is on for a TTY unless NO_COLOR is set to a non-empty value. Hyperlinks follow a reduced port of the
|
|
9
|
+
# decision order of the npm package supports-hyperlinks: an allowlist of terminals known to
|
|
10
|
+
# render them, so older terminals never print escape junk. Colour and hyperlinks are decided
|
|
11
|
+
# independently, and `--color` / `--hyperlinks` (always, never) override each guess.
|
|
12
|
+
class Terminal
|
|
13
|
+
STYLES = { error: '1;31', warning: '1;33', secondary: '34' }.freeze
|
|
14
|
+
|
|
15
|
+
# Minimum TERM_PROGRAM_VERSION per TERM_PROGRAM; nil means any version.
|
|
16
|
+
PROGRAMS = { 'iTerm.app' => [3, 1], 'WezTerm' => [20_200_620], 'vscode' => [1, 72], 'ghostty' => nil,
|
|
17
|
+
'zed' => nil, 'Orca' => nil, 'WarpTerminal' => nil }.freeze
|
|
18
|
+
# VTE 0.50.0 crashes on hyperlinks.
|
|
19
|
+
VTE_MINIMUM = [0, 50, 1].freeze
|
|
20
|
+
TERMS = %w[alacritty xterm-kitty].freeze
|
|
21
|
+
|
|
22
|
+
# +env+ is a Hash-like environment; +tty+ says whether stdout is a terminal.
|
|
23
|
+
def initialize(env:, tty:, color: 'auto', hyperlinks: 'auto')
|
|
24
|
+
@env = env
|
|
25
|
+
@tty = tty
|
|
26
|
+
@color = decide(color) { tty && env['NO_COLOR'].to_s.empty? }
|
|
27
|
+
@hyperlinks = decide(hyperlinks) { detect_hyperlinks }
|
|
28
|
+
end
|
|
29
|
+
|
|
30
|
+
def color? = @color
|
|
31
|
+
|
|
32
|
+
def hyperlinks? = @hyperlinks
|
|
33
|
+
|
|
34
|
+
# +text+ in the SGR style named +style+ (a key of STYLES) when colour is on.
|
|
35
|
+
def paint(text, style) = color? ? "\e[#{STYLES.fetch(style)}m#{text}\e[0m" : text
|
|
36
|
+
|
|
37
|
+
# +text+ as an OSC 8 hyperlink to +url+ when hyperlinks are on.
|
|
38
|
+
def link(text, url) = hyperlinks? ? "\e]8;;#{url}\e\\#{text}\e]8;;\e\\" : text
|
|
39
|
+
|
|
40
|
+
private
|
|
41
|
+
|
|
42
|
+
def decide(option)
|
|
43
|
+
case option
|
|
44
|
+
when 'always' then true
|
|
45
|
+
when 'never' then false
|
|
46
|
+
else yield
|
|
47
|
+
end
|
|
48
|
+
end
|
|
49
|
+
|
|
50
|
+
# First match wins: FORCE_HYPERLINK, then no TTY or CI, then known terminals.
|
|
51
|
+
def detect_hyperlinks
|
|
52
|
+
force = @env['FORCE_HYPERLINK'].to_s
|
|
53
|
+
return force != '0' unless force.empty?
|
|
54
|
+
return false unless @tty && !ci?
|
|
55
|
+
|
|
56
|
+
@env.key?('WT_SESSION') || program? || vte? || TERMS.include?(@env['TERM'])
|
|
57
|
+
end
|
|
58
|
+
|
|
59
|
+
def ci? = @env.key?('CI') || @env.key?('TEAMCITY_VERSION')
|
|
60
|
+
|
|
61
|
+
def program?
|
|
62
|
+
name = @env['TERM_PROGRAM']
|
|
63
|
+
return false unless PROGRAMS.key?(name)
|
|
64
|
+
|
|
65
|
+
minimum = PROGRAMS[name]
|
|
66
|
+
minimum.nil? || at_least?(version(@env['TERM_PROGRAM_VERSION']), minimum)
|
|
67
|
+
end
|
|
68
|
+
|
|
69
|
+
# VTE_VERSION is either dotted (0.50.1) or packed as major * 10000 + minor * 100 + micro (5001).
|
|
70
|
+
def vte?
|
|
71
|
+
value = @env['VTE_VERSION'].to_s
|
|
72
|
+
parts = value.include?('.') ? version(value) : unpack(value)
|
|
73
|
+
at_least?(parts, VTE_MINIMUM)
|
|
74
|
+
end
|
|
75
|
+
|
|
76
|
+
def unpack(value)
|
|
77
|
+
return unless value.match?(/\A\d+\z/)
|
|
78
|
+
|
|
79
|
+
number = value.to_i
|
|
80
|
+
[number / 10_000, number / 100 % 100, number % 100]
|
|
81
|
+
end
|
|
82
|
+
|
|
83
|
+
# The leading dotted number: `1.95.0-insider` is [1, 95, 0], `20200620-160318-e00b076c` is [20200620].
|
|
84
|
+
def version(value) = value.to_s[/\A\d+(\.\d+)*/]&.split('.')&.map(&:to_i)
|
|
85
|
+
|
|
86
|
+
def at_least?(parts, minimum)
|
|
87
|
+
return false if parts.nil?
|
|
88
|
+
|
|
89
|
+
size = [parts.size, minimum.size].max
|
|
90
|
+
(parts.fill(0, parts.size...size) <=> minimum.dup.fill(0, minimum.size...size)) >= 0
|
|
91
|
+
end
|
|
92
|
+
|
|
93
|
+
# Neither colour nor hyperlinks.
|
|
94
|
+
PLAIN = new(env: {}, tty: false)
|
|
95
|
+
end
|
|
96
|
+
end
|
|
97
|
+
end
|
|
@@ -0,0 +1,146 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Scully
|
|
4
|
+
module Render
|
|
5
|
+
# Renders `scully why`: the finding block followed by its explanation.
|
|
6
|
+
#
|
|
7
|
+
# <finding block, as in the report>
|
|
8
|
+
#
|
|
9
|
+
# because:
|
|
10
|
+
# <the rule's rationale>
|
|
11
|
+
#
|
|
12
|
+
# what happened:
|
|
13
|
+
# <the rule's explain lines>
|
|
14
|
+
#
|
|
15
|
+
# help:
|
|
16
|
+
# <the rule's fix guidance>
|
|
17
|
+
#
|
|
18
|
+
# evidence: <the file's status>
|
|
19
|
+
# see also: <companion cops>
|
|
20
|
+
class Why
|
|
21
|
+
# Longest line, indent included: wrapped text stays shorter than 92 columns.
|
|
22
|
+
WIDTH = 91
|
|
23
|
+
INDENT = ' '
|
|
24
|
+
COMPLETE = 'evidence: static analysis (complete file)'
|
|
25
|
+
SEE_ALSO = 'see also: '
|
|
26
|
+
NO_COMPANION = 'no matching cop in rubocop-rspec or rubocop-minitest'
|
|
27
|
+
URL_INDENT = ' ' * 12
|
|
28
|
+
DETAILS = 'Use `scully why <location>` for details:'
|
|
29
|
+
OTHERS = 'Findings in this file:'
|
|
30
|
+
COLUMN_GAP = ' '
|
|
31
|
+
|
|
32
|
+
# +file+ is a FileReport and +findings+ its findings in report order; +rules+ maps rule ids
|
|
33
|
+
# to rules; +source+ responds to line(number); +terminal+ adds colour and hyperlinks when
|
|
34
|
+
# they are on.
|
|
35
|
+
def initialize(file, findings:, rules:, source:, terminal: Terminal::PLAIN)
|
|
36
|
+
@file = file
|
|
37
|
+
@findings = findings
|
|
38
|
+
@rules = rules
|
|
39
|
+
@source = source
|
|
40
|
+
@terminal = terminal
|
|
41
|
+
end
|
|
42
|
+
|
|
43
|
+
# Every finding whose primary span starts on +line+, by column then rule id, explained in
|
|
44
|
+
# full and separated by blank lines; nil when there is none.
|
|
45
|
+
def location(line)
|
|
46
|
+
found = @findings.select { it.primary.span.start_line == line }
|
|
47
|
+
return if found.empty?
|
|
48
|
+
|
|
49
|
+
found.sort_by { [it.primary.span.start_column, it.rule_id.to_s] }.map { explanation(it) }.join("\n")
|
|
50
|
+
end
|
|
51
|
+
|
|
52
|
+
# The file's findings, one row each, then a `scully why` command for each.
|
|
53
|
+
def index
|
|
54
|
+
return "#{@file.path}#{status}: no findings\n" if @findings.empty?
|
|
55
|
+
|
|
56
|
+
"#{@file.path}#{status}\n\n#{table(rows).join}\n#{DETAILS}\n\n#{commands.join}"
|
|
57
|
+
end
|
|
58
|
+
|
|
59
|
+
# The report's skip line when the file was skipped, so it never looks clean; nil otherwise.
|
|
60
|
+
def skipped = (Report.skip_line(@file) if @file.status == :skipped)
|
|
61
|
+
|
|
62
|
+
# The message for a +line+ without findings, pointing at the file's findings instead.
|
|
63
|
+
def missing(line)
|
|
64
|
+
others = @findings.empty? ? "No findings in this file.\n" : "#{OTHERS}\n#{commands(" ").join}"
|
|
65
|
+
"scully: no finding at #{@file.path}:#{line}\n#{others}"
|
|
66
|
+
end
|
|
67
|
+
|
|
68
|
+
# Wraps +text+ greedily on spaces so no line exceeds +width+ characters, prefix included.
|
|
69
|
+
# The first line starts with +first+, the others with +indent+. A word longer than the
|
|
70
|
+
# width is never broken. Returns the lines without terminators.
|
|
71
|
+
def self.wrap(text, indent:, first: indent, width: WIDTH)
|
|
72
|
+
rows = [[first]]
|
|
73
|
+
text.split(' ').each do |word|
|
|
74
|
+
row = rows.last
|
|
75
|
+
rows << (row = [indent]) if row.size > 1 && "#{row.first}#{[*row.drop(1), word].join(" ")}".size > width
|
|
76
|
+
row << word
|
|
77
|
+
end
|
|
78
|
+
rows.map { |prefix, *words| "#{prefix}#{words.join(" ")}" }
|
|
79
|
+
end
|
|
80
|
+
|
|
81
|
+
private
|
|
82
|
+
|
|
83
|
+
def status = @file.status == :incomplete ? Snippet::INCONCLUSIVE : ''
|
|
84
|
+
|
|
85
|
+
def message(finding) = @rules.fetch(finding.rule_id).class.message
|
|
86
|
+
|
|
87
|
+
# Each column but the last padded to its widest cell.
|
|
88
|
+
def table(rows)
|
|
89
|
+
widths = rows.transpose[0...-1].map { |cells| cells.map(&:size).max }
|
|
90
|
+
rows.map do |*cells, last|
|
|
91
|
+
"#{INDENT}#{cells.zip(widths).map { |cell, width| cell.ljust(width) }.join(COLUMN_GAP)}#{COLUMN_GAP}#{last}\n"
|
|
92
|
+
end
|
|
93
|
+
end
|
|
94
|
+
|
|
95
|
+
def rows
|
|
96
|
+
@findings.map do
|
|
97
|
+
span = it.primary.span
|
|
98
|
+
["#{it.severity}[#{it.rule_id}]", "#{span.start_line}:#{span.start_column}", message(it)]
|
|
99
|
+
end
|
|
100
|
+
end
|
|
101
|
+
|
|
102
|
+
def commands(indent = '') = Report.commands(@findings).map { "#{indent}#{it}\n" }
|
|
103
|
+
|
|
104
|
+
def explanation(finding)
|
|
105
|
+
block = Snippet.render(finding, message: message(finding), source: @source, terminal: @terminal)
|
|
106
|
+
rule = @rules.fetch(finding.rule_id)
|
|
107
|
+
declaration = rule.class
|
|
108
|
+
lines = [*section('because', [declaration.because]), *section('what happened', rule.explain(finding)),
|
|
109
|
+
*section('help', [declaration.help]), '', *evidence, *see_also(declaration.companions)]
|
|
110
|
+
"#{block}#{lines.map { "#{it}\n" }.join}"
|
|
111
|
+
end
|
|
112
|
+
|
|
113
|
+
def section(title, paragraphs) = ['', "#{title}:", *paragraphs.flat_map { self.class.wrap(it, indent: INDENT) }]
|
|
114
|
+
|
|
115
|
+
# The file's status, not the finding's: a parse finding is certain, but its file is not.
|
|
116
|
+
def evidence
|
|
117
|
+
lines = @file.parse_errors.map { it.span.start_line }.uniq
|
|
118
|
+
return [COMPLETE] if lines.empty?
|
|
119
|
+
|
|
120
|
+
self.class.wrap("inconclusive. This file has parse errors on #{line_list(lines)}, so parts of it may not " \
|
|
121
|
+
'have been analyzed. Fix them and run again.', first: 'evidence: ', indent: INDENT)
|
|
122
|
+
end
|
|
123
|
+
|
|
124
|
+
def line_list(lines)
|
|
125
|
+
return "line #{lines.first}" if lines.one?
|
|
126
|
+
|
|
127
|
+
"lines #{lines[0...-1].join(", ")} and #{lines.last}"
|
|
128
|
+
end
|
|
129
|
+
|
|
130
|
+
def see_also(companions)
|
|
131
|
+
return ["#{SEE_ALSO}#{NO_COMPANION}"] if companions.empty?
|
|
132
|
+
|
|
133
|
+
companions.each_with_index.flat_map do |companion, index|
|
|
134
|
+
prefix = index.zero? ? SEE_ALSO : ' ' * SEE_ALSO.size
|
|
135
|
+
url = "#{URL_INDENT}#{companion.url}" unless @terminal.hyperlinks?
|
|
136
|
+
["#{prefix}#{companion_text(companion)}", *url]
|
|
137
|
+
end
|
|
138
|
+
end
|
|
139
|
+
|
|
140
|
+
def companion_text(companion)
|
|
141
|
+
state = ', disabled by default' unless companion.default_enabled
|
|
142
|
+
"#{@terminal.link(companion.cop, companion.url)} (#{companion.plugin}#{state})"
|
|
143
|
+
end
|
|
144
|
+
end
|
|
145
|
+
end
|
|
146
|
+
end
|
data/lib/scully/rule.rb
ADDED
|
@@ -0,0 +1,85 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Scully
|
|
4
|
+
# A RuboCop cop that covers the same ground as a rule.
|
|
5
|
+
Companion = Data.define(:cop, :plugin, :url, :default_enabled)
|
|
6
|
+
|
|
7
|
+
# Base class for rules. A subclass declares its fields with the class-level DSL:
|
|
8
|
+
#
|
|
9
|
+
# class NoAssertion < Scully::Rule
|
|
10
|
+
# id :no_assertion
|
|
11
|
+
# severity :warning
|
|
12
|
+
# message 'test makes no assertion'
|
|
13
|
+
# because '...'
|
|
14
|
+
# help '...'
|
|
15
|
+
# companions Companion.new(...), Companion.new(...) # or: companions :none
|
|
16
|
+
#
|
|
17
|
+
# def check(test_case) = [...] # -> Array<Finding>
|
|
18
|
+
# def explain(finding) = [...] # -> Array<String>
|
|
19
|
+
# end
|
|
20
|
+
#
|
|
21
|
+
# A subclass that omits a field raises DeclarationError when its class body ends.
|
|
22
|
+
class Rule
|
|
23
|
+
class DeclarationError < Scully::Error; end
|
|
24
|
+
|
|
25
|
+
FIELDS = %i[id severity message because help companions].freeze
|
|
26
|
+
|
|
27
|
+
class << self
|
|
28
|
+
def inherited(subclass)
|
|
29
|
+
super
|
|
30
|
+
trace = TracePoint.new(:end) do |event|
|
|
31
|
+
next unless event.self.equal?(subclass)
|
|
32
|
+
|
|
33
|
+
trace.disable
|
|
34
|
+
subclass.validate!
|
|
35
|
+
end
|
|
36
|
+
trace.enable
|
|
37
|
+
end
|
|
38
|
+
|
|
39
|
+
def id(value = nil) = declare(:id, value)
|
|
40
|
+
def severity(value = nil) = declare(:severity, value)
|
|
41
|
+
def message(value = nil) = declare(:message, value)
|
|
42
|
+
def because(value = nil) = declare(:because, value)
|
|
43
|
+
def help(value = nil) = declare(:help, value)
|
|
44
|
+
|
|
45
|
+
# Companion cops, or :none when no cop matches. Reads return an Array.
|
|
46
|
+
def companions(*values)
|
|
47
|
+
return declare(:companions, values == [:none] ? [] : values) unless values.empty?
|
|
48
|
+
|
|
49
|
+
declared(:companions)
|
|
50
|
+
end
|
|
51
|
+
|
|
52
|
+
def validate!
|
|
53
|
+
missing = FIELDS.reject { instance_variable_defined?(:"@#{it}") }
|
|
54
|
+
raise DeclarationError, "#{name || "rule"} does not declare #{missing.join(", ")}" unless missing.empty?
|
|
55
|
+
raise DeclarationError, "#{id}: unknown severity #{severity.inspect}" unless SEVERITIES.include?(severity)
|
|
56
|
+
return if companions.all?(Companion)
|
|
57
|
+
|
|
58
|
+
raise DeclarationError, "#{id}: companions must be Companion values or :none"
|
|
59
|
+
end
|
|
60
|
+
|
|
61
|
+
private
|
|
62
|
+
|
|
63
|
+
def declare(field, value)
|
|
64
|
+
return declared(field) if value.nil?
|
|
65
|
+
|
|
66
|
+
instance_variable_set(:"@#{field}", value)
|
|
67
|
+
end
|
|
68
|
+
|
|
69
|
+
def declared(field) = instance_variable_get(:"@#{field}")
|
|
70
|
+
end
|
|
71
|
+
|
|
72
|
+
def check(_test_case) = raise(NotImplementedError, "#{self.class} must implement #check")
|
|
73
|
+
|
|
74
|
+
def explain(_finding) = raise(NotImplementedError, "#{self.class} must implement #explain")
|
|
75
|
+
|
|
76
|
+
private
|
|
77
|
+
|
|
78
|
+
# Builds a finding for this rule. Evidence starts :static; the analyzer downgrades it
|
|
79
|
+
# for incomplete files.
|
|
80
|
+
def finding(primary:, test_name:, secondary: [], facts: {})
|
|
81
|
+
Finding.new(rule_id: self.class.id, severity: self.class.severity, primary:, secondary:,
|
|
82
|
+
test_name:, facts:, evidence: :static)
|
|
83
|
+
end
|
|
84
|
+
end
|
|
85
|
+
end
|
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Scully
|
|
4
|
+
module Rules
|
|
5
|
+
# Reports a test that runs code but never checks a result.
|
|
6
|
+
class NoAssertion < Rule
|
|
7
|
+
id :no_assertion
|
|
8
|
+
severity :warning
|
|
9
|
+
message 'test makes no assertion'
|
|
10
|
+
because 'A test with no assertion passes whatever the code does, so it gives coverage without evidence.'
|
|
11
|
+
help 'Add an assertion that states what the test claims. If an assertion lives in a helper defined ' \
|
|
12
|
+
'in another file, scully cannot see it yet and this finding may be a false positive.'
|
|
13
|
+
companions Companion.new(cop: 'RSpec/NoExpectationExample', plugin: 'rubocop-rspec',
|
|
14
|
+
url: 'https://docs.rubocop.org/rubocop-rspec/latest/cops_rspec.html#rspecnoexpectationexample',
|
|
15
|
+
default_enabled: true),
|
|
16
|
+
Companion.new(cop: 'Minitest/NoAssertions', plugin: 'rubocop-minitest',
|
|
17
|
+
url: 'https://docs.rubocop.org/rubocop-minitest/latest/cops_minitest.html#minitestnoassertions',
|
|
18
|
+
default_enabled: false)
|
|
19
|
+
|
|
20
|
+
LABEL = 'this test runs code but never checks a result'
|
|
21
|
+
# How a candidate helper was resolved, for helpers that did not count as assertions.
|
|
22
|
+
RESOLUTIONS = { same_file_without_assertion: 'defined in this file, no assertions',
|
|
23
|
+
unresolved: 'not defined in this file, not followed' }.freeze
|
|
24
|
+
LISTED_HELPERS = 5
|
|
25
|
+
|
|
26
|
+
def check(test_case)
|
|
27
|
+
return [] unless test_case.status == :active && test_case.assertions.empty?
|
|
28
|
+
|
|
29
|
+
[finding(primary: Label.new(span: test_case.span, text: LABEL), test_name: test_case.name,
|
|
30
|
+
facts: { helper_calls: test_case.helper_calls.map { [it.name, it.resolution] } })]
|
|
31
|
+
end
|
|
32
|
+
|
|
33
|
+
def explain(finding)
|
|
34
|
+
["No assertion, expectation, or assertion helper was found in this test's body.",
|
|
35
|
+
"Calls checked as possible helpers: #{helpers(finding.facts[:helper_calls])}."]
|
|
36
|
+
end
|
|
37
|
+
|
|
38
|
+
private
|
|
39
|
+
|
|
40
|
+
# Each helper once, at its first call: up to LISTED_HELPERS, then a count of the rest.
|
|
41
|
+
def helpers(calls)
|
|
42
|
+
calls = calls.uniq(&:first)
|
|
43
|
+
return 'none' if calls.empty?
|
|
44
|
+
|
|
45
|
+
items = calls.first(LISTED_HELPERS).map { |name, resolution| "`#{name}` (#{RESOLUTIONS.fetch(resolution)})" }
|
|
46
|
+
items << "#{calls.size - LISTED_HELPERS} more" if calls.size > LISTED_HELPERS
|
|
47
|
+
items.size == 1 ? items.first : "#{items[0...-1].join(", ")} and #{items.last}"
|
|
48
|
+
end
|
|
49
|
+
end
|
|
50
|
+
end
|
|
51
|
+
end
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Scully
|
|
4
|
+
module Rules
|
|
5
|
+
# Reports the syntax errors Prism found in a file, one finding per line that has any.
|
|
6
|
+
#
|
|
7
|
+
# Built in: the analyzer runs it on every file report, not on test cases.
|
|
8
|
+
class Parse < Rule
|
|
9
|
+
id :parse
|
|
10
|
+
severity :error
|
|
11
|
+
message 'syntax error'
|
|
12
|
+
because 'Scully can only check code it can read. A file with syntax errors is analyzed only in part, ' \
|
|
13
|
+
'so its results are inconclusive.'
|
|
14
|
+
help 'Fix the syntax error. `ruby -c <path>` shows Ruby\'s own message.'
|
|
15
|
+
companions :none
|
|
16
|
+
|
|
17
|
+
def check(_test_case) = []
|
|
18
|
+
|
|
19
|
+
# Every message Prism gave for the finding's line.
|
|
20
|
+
def explain(finding) = finding.facts[:messages]
|
|
21
|
+
|
|
22
|
+
# +file+'s parse errors are already sorted by line, column, then message bytes, so the
|
|
23
|
+
# first error on each line labels the finding and the rest are counted.
|
|
24
|
+
def findings(file)
|
|
25
|
+
file.parse_errors.group_by { it.span.start_line }.values.map do |errors|
|
|
26
|
+
first = errors.first
|
|
27
|
+
more = errors.size > 1 ? " (+#{errors.size - 1} more)" : ''
|
|
28
|
+
finding(primary: Label.new(span: first.span, text: "#{first.message}#{more}"), test_name: nil,
|
|
29
|
+
facts: { messages: errors.map(&:message) })
|
|
30
|
+
end
|
|
31
|
+
end
|
|
32
|
+
end
|
|
33
|
+
end
|
|
34
|
+
end
|
|
@@ -0,0 +1,101 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require 'prism'
|
|
4
|
+
|
|
5
|
+
module Scully
|
|
6
|
+
module Rules
|
|
7
|
+
# Reports an assertion that compares a stubbed call with the value the stub returns.
|
|
8
|
+
class Tautology < Rule
|
|
9
|
+
id :tautology
|
|
10
|
+
severity :error
|
|
11
|
+
message 'assertion checks a value the test stubbed'
|
|
12
|
+
because 'The assertion checks a value the test itself stubbed, so it can never fail and says nothing ' \
|
|
13
|
+
'about the code under test.'
|
|
14
|
+
help 'Assert on the result of the code that uses the stubbed object, not on the stub itself. If the ' \
|
|
15
|
+
'test only documents the stub, delete it.'
|
|
16
|
+
companions :none
|
|
17
|
+
|
|
18
|
+
LABEL = 'compares the stub with its own return value'
|
|
19
|
+
PLACES = { setup: 'in setup', body: 'here' }.freeze
|
|
20
|
+
|
|
21
|
+
def check(test_case)
|
|
22
|
+
return [] unless test_case.status == :active
|
|
23
|
+
|
|
24
|
+
test_case.assertions.filter_map { tautology(it, test_case) }
|
|
25
|
+
end
|
|
26
|
+
|
|
27
|
+
def explain(finding)
|
|
28
|
+
facts = finding.facts
|
|
29
|
+
["`#{facts[:subject]}` is stubbed to return `#{facts[:value]}` (line #{facts[:stub_line]}), and the " \
|
|
30
|
+
"expectation on line #{facts[:assertion_line]} compares `#{facts[:subject]}` with `#{facts[:value]}`. " \
|
|
31
|
+
'No code under test runs between the stub and the assertion.']
|
|
32
|
+
end
|
|
33
|
+
|
|
34
|
+
private
|
|
35
|
+
|
|
36
|
+
def tautology(assertion, test_case)
|
|
37
|
+
call = stubbed_call(assertion) or return
|
|
38
|
+
double = nearest(test_case.doubles, call, assertion.span)
|
|
39
|
+
return unless double&.stub_source && double.returns && same?(double.returns, assertion.expected)
|
|
40
|
+
|
|
41
|
+
report(assertion, double, test_case)
|
|
42
|
+
end
|
|
43
|
+
|
|
44
|
+
# The `R.m` an equality assertion checks: a call with a receiver and no arguments.
|
|
45
|
+
def stubbed_call(assertion)
|
|
46
|
+
call = assertion.subject
|
|
47
|
+
call if assertion.comparison && call.is_a?(Prism::CallNode) && call.receiver &&
|
|
48
|
+
call.arguments.nil? && call.block.nil?
|
|
49
|
+
end
|
|
50
|
+
|
|
51
|
+
# The last double for the same receiver and method that is visible at the assertion: a later
|
|
52
|
+
# stub replaces an earlier one, so only the nearest one decides.
|
|
53
|
+
def nearest(doubles, call, at)
|
|
54
|
+
doubles.select { it.method_name == call.name && same_receiver?(it.receiver, call.receiver) && visible?(it, at) }
|
|
55
|
+
.last
|
|
56
|
+
end
|
|
57
|
+
|
|
58
|
+
# Bare identifiers (a local variable, or a `let` read as a method call) match by name;
|
|
59
|
+
# anything else must be structurally equal.
|
|
60
|
+
def same_receiver?(left, right)
|
|
61
|
+
names = [identifier(left), identifier(right)]
|
|
62
|
+
names.all? ? names.uniq.size == 1 : same?(left, right)
|
|
63
|
+
end
|
|
64
|
+
|
|
65
|
+
# Prism's `Node#===` compares structure and ignores source positions, so `tax . rate`
|
|
66
|
+
# equals `tax.rate`; `==` would compare identity.
|
|
67
|
+
def same?(left, right)
|
|
68
|
+
left === right # rubocop:disable Style/CaseEquality
|
|
69
|
+
end
|
|
70
|
+
|
|
71
|
+
def identifier(node)
|
|
72
|
+
case node
|
|
73
|
+
when Prism::LocalVariableReadNode then node.name
|
|
74
|
+
when Prism::CallNode then node.name if node.receiver.nil? && node.arguments.nil? && node.block.nil?
|
|
75
|
+
end
|
|
76
|
+
end
|
|
77
|
+
|
|
78
|
+
# Setup runs before the body; a body stub must come before the assertion; a block stub
|
|
79
|
+
# only applies inside its block.
|
|
80
|
+
def visible?(double, at)
|
|
81
|
+
(double.origin == :setup || (position(double.span) <=> position(at)).negative?) &&
|
|
82
|
+
(double.scope.nil? || inside?(at, double.scope))
|
|
83
|
+
end
|
|
84
|
+
|
|
85
|
+
def inside?(span, scope)
|
|
86
|
+
(position(span) <=> position(scope)) >= 0 && (position(span) <=> [scope.end_line, scope.end_column]).negative?
|
|
87
|
+
end
|
|
88
|
+
|
|
89
|
+
def position(span) = [span.start_line, span.start_column]
|
|
90
|
+
|
|
91
|
+
def report(assertion, double, test_case)
|
|
92
|
+
value = double.returns.slice
|
|
93
|
+
secondary = Label.new(span: double.span, text: "stubbed #{PLACES.fetch(double.origin)} to return `#{value}`")
|
|
94
|
+
facts = { stub_line: double.span.start_line, value:, subject: assertion.subject.slice,
|
|
95
|
+
assertion_line: assertion.span.start_line }
|
|
96
|
+
finding(primary: Label.new(span: assertion.span, text: LABEL), secondary: [secondary],
|
|
97
|
+
test_name: test_case.name, facts:)
|
|
98
|
+
end
|
|
99
|
+
end
|
|
100
|
+
end
|
|
101
|
+
end
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require 'prism'
|
|
4
|
+
|
|
5
|
+
module Scully
|
|
6
|
+
# A test file read from disk and parsed with Prism.
|
|
7
|
+
#
|
|
8
|
+
# A file is incomplete when it could not be read, is not valid UTF-8, or has syntax errors.
|
|
9
|
+
# Unreadable and invalid files have no parse result; +problem+ says why.
|
|
10
|
+
class SourceFile
|
|
11
|
+
attr_reader :path, :source, :result, :parse_errors, :problem
|
|
12
|
+
|
|
13
|
+
# Reads +path+, resolved against +base+ when given; spans keep +path+ as written.
|
|
14
|
+
def self.read(path, base: nil)
|
|
15
|
+
bytes = File.binread(base ? File.join(base, path) : path)
|
|
16
|
+
source = bytes.force_encoding(Encoding::UTF_8)
|
|
17
|
+
return new(path, problem: 'file is not valid UTF-8') unless source.valid_encoding?
|
|
18
|
+
|
|
19
|
+
new(path, source:)
|
|
20
|
+
rescue SystemCallError => e
|
|
21
|
+
new(path, problem: "could not read file (#{e.class.name.split("::").last})")
|
|
22
|
+
end
|
|
23
|
+
|
|
24
|
+
def initialize(path, source: nil, problem: nil)
|
|
25
|
+
@path = path
|
|
26
|
+
@source = source
|
|
27
|
+
@problem = problem
|
|
28
|
+
@result = source && Prism.parse(source, filepath: path)
|
|
29
|
+
@parse_errors = @result ? collect_errors(@result.errors) : []
|
|
30
|
+
@lines = source ? source.lines.map { it.chomp.delete_suffix("\r") } : []
|
|
31
|
+
end
|
|
32
|
+
|
|
33
|
+
def incomplete? = !problem.nil? || !parse_errors.empty?
|
|
34
|
+
|
|
35
|
+
# The text of a 1-based line, without its line terminator.
|
|
36
|
+
def line(number) = @lines.fetch(number - 1, '')
|
|
37
|
+
|
|
38
|
+
# Converts a Prism location to a Span with 1-based character columns. End is exclusive.
|
|
39
|
+
def span(location)
|
|
40
|
+
Span.new(path:,
|
|
41
|
+
start_line: location.start_line, start_column: location.start_character_column + 1,
|
|
42
|
+
end_line: location.end_line, end_column: location.end_character_column + 1)
|
|
43
|
+
end
|
|
44
|
+
|
|
45
|
+
private
|
|
46
|
+
|
|
47
|
+
def collect_errors(errors)
|
|
48
|
+
errors.map { ParseError.new(span: span(it.location), message: it.message) }
|
|
49
|
+
.uniq { [it.span.start_line, it.span.start_column, it.message] }
|
|
50
|
+
.sort_by { [*it.span.sort_key, it.message.b] }
|
|
51
|
+
end
|
|
52
|
+
end
|
|
53
|
+
end
|
data/lib/scully/version.rb
CHANGED
data/lib/scully.rb
CHANGED
|
@@ -4,5 +4,22 @@ require_relative 'scully/version'
|
|
|
4
4
|
|
|
5
5
|
module Scully
|
|
6
6
|
class Error < StandardError; end
|
|
7
|
-
# Your code goes here...
|
|
8
7
|
end
|
|
8
|
+
|
|
9
|
+
require_relative 'scully/model'
|
|
10
|
+
require_relative 'scully/discovery'
|
|
11
|
+
require_relative 'scully/source_file'
|
|
12
|
+
require_relative 'scully/frameworks/detector'
|
|
13
|
+
require_relative 'scully/frameworks/helpers'
|
|
14
|
+
require_relative 'scully/frameworks/rspec'
|
|
15
|
+
require_relative 'scully/frameworks/minitest'
|
|
16
|
+
require_relative 'scully/rule'
|
|
17
|
+
require_relative 'scully/rules/no_assertion'
|
|
18
|
+
require_relative 'scully/rules/tautology'
|
|
19
|
+
require_relative 'scully/rules/parse'
|
|
20
|
+
require_relative 'scully/analyzer'
|
|
21
|
+
require_relative 'scully/render/terminal'
|
|
22
|
+
require_relative 'scully/render/snippet'
|
|
23
|
+
require_relative 'scully/render/report'
|
|
24
|
+
require_relative 'scully/render/why'
|
|
25
|
+
require_relative 'scully/cli'
|
metadata
CHANGED
|
@@ -1,17 +1,34 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: scully
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.0
|
|
4
|
+
version: 0.1.0
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Theo Melo
|
|
8
8
|
bindir: exe
|
|
9
9
|
cert_chain: []
|
|
10
10
|
date: 1980-01-02 00:00:00.000000000 Z
|
|
11
|
-
dependencies:
|
|
11
|
+
dependencies:
|
|
12
|
+
- !ruby/object:Gem::Dependency
|
|
13
|
+
name: prism
|
|
14
|
+
requirement: !ruby/object:Gem::Requirement
|
|
15
|
+
requirements:
|
|
16
|
+
- - ">="
|
|
17
|
+
- !ruby/object:Gem::Version
|
|
18
|
+
version: '1.8'
|
|
19
|
+
type: :runtime
|
|
20
|
+
prerelease: false
|
|
21
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
22
|
+
requirements:
|
|
23
|
+
- - ">="
|
|
24
|
+
- !ruby/object:Gem::Version
|
|
25
|
+
version: '1.8'
|
|
26
|
+
description: Scully statically checks RSpec and Minitest suites with Prism, without
|
|
27
|
+
running them, and reports tests that check nothing or can never fail.
|
|
12
28
|
email:
|
|
13
29
|
- 9499829+theomelo@users.noreply.github.com
|
|
14
|
-
executables:
|
|
30
|
+
executables:
|
|
31
|
+
- scully
|
|
15
32
|
extensions: []
|
|
16
33
|
extra_rdoc_files: []
|
|
17
34
|
files:
|
|
@@ -20,17 +37,33 @@ files:
|
|
|
20
37
|
- LICENSE.txt
|
|
21
38
|
- README.md
|
|
22
39
|
- Rakefile
|
|
40
|
+
- exe/scully
|
|
23
41
|
- lib/scully.rb
|
|
42
|
+
- lib/scully/analyzer.rb
|
|
43
|
+
- lib/scully/cli.rb
|
|
44
|
+
- lib/scully/discovery.rb
|
|
45
|
+
- lib/scully/frameworks/detector.rb
|
|
46
|
+
- lib/scully/frameworks/helpers.rb
|
|
47
|
+
- lib/scully/frameworks/minitest.rb
|
|
48
|
+
- lib/scully/frameworks/rspec.rb
|
|
49
|
+
- lib/scully/model.rb
|
|
50
|
+
- lib/scully/render/report.rb
|
|
51
|
+
- lib/scully/render/snippet.rb
|
|
52
|
+
- lib/scully/render/terminal.rb
|
|
53
|
+
- lib/scully/render/why.rb
|
|
54
|
+
- lib/scully/rule.rb
|
|
55
|
+
- lib/scully/rules/no_assertion.rb
|
|
56
|
+
- lib/scully/rules/parse.rb
|
|
57
|
+
- lib/scully/rules/tautology.rb
|
|
58
|
+
- lib/scully/source_file.rb
|
|
24
59
|
- lib/scully/version.rb
|
|
25
60
|
- sig/scully.rbs
|
|
26
|
-
- sig/test_scully.rbs
|
|
27
61
|
homepage: https://github.com/theomelo/scully
|
|
28
62
|
licenses:
|
|
29
63
|
- MIT
|
|
30
64
|
metadata:
|
|
31
65
|
allowed_push_host: https://rubygems.org
|
|
32
66
|
homepage_uri: https://github.com/theomelo/scully
|
|
33
|
-
source_code_uri: https://github.com/theomelo/scully
|
|
34
67
|
changelog_uri: https://github.com/theomelo/scully/blob/main/CHANGELOG.md
|
|
35
68
|
rubygems_mfa_required: 'true'
|
|
36
69
|
rdoc_options: []
|
|
@@ -49,5 +82,5 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
|
49
82
|
requirements: []
|
|
50
83
|
rubygems_version: 4.0.16
|
|
51
84
|
specification_version: 4
|
|
52
|
-
summary:
|
|
85
|
+
summary: Finds Ruby tests that cannot be trusted as evidence
|
|
53
86
|
test_files: []
|