ndr_dev_support 5.9.0 → 5.10.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/CHANGELOG.md +9 -0
- data/README.md +17 -1
- data/code_safety.yml +21 -13
- data/lib/ndr_dev_support/integration_testing/flakey_tests.rb +6 -0
- data/lib/ndr_dev_support/rubocop/executor.rb +2 -2
- data/lib/ndr_dev_support/rubocop/inject.rb +9 -1
- data/lib/ndr_dev_support/rubocop/range_finder.rb +0 -1
- data/lib/ndr_dev_support/version.rb +1 -1
- data/lib/tasks/audit_code.rake +203 -134
- data/ndr_dev_support.gemspec +1 -3
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 918c657fc342d3fa170cdaa1a8ed1ae2f749f5540c97c3fe4559f96286a72c78
|
4
|
+
data.tar.gz: 5f1ee7f3b4e3d4b2c84548b67edcd90f21eb71a69cd10e3c3141bb3ba934d156
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 62006069af4083dea2856c0d911aff1a9aacc209074a96269d18cfd7c64af6ac538d88bee4adee853872ebe39108e7fcbf0f5766d189f3f959fd05cb14aa2115
|
7
|
+
data.tar.gz: a58e4bb49212b97f68a0683dc2dd3c014573416a571950a930f6c73aa3eb8ba828a3d1db669d61aa7bfa11f5d0133bfdb5c33b87f112355da399291195f3c7fd
|
data/CHANGELOG.md
CHANGED
@@ -1,6 +1,15 @@
|
|
1
1
|
## [Unreleased]
|
2
2
|
*no unreleased changes*
|
3
3
|
|
4
|
+
## 5.10.0 / 2021-01-29
|
5
|
+
### Added
|
6
|
+
* Added `test_repeatedly` for integration test debugging (#85)
|
7
|
+
* Added `outfile` and `filter` options to code auditing (#93)
|
8
|
+
|
9
|
+
### Fixed
|
10
|
+
* Fixed excessive `parser/current` warnings (#91)
|
11
|
+
* Improved performance of interactive code auditing (#93)
|
12
|
+
|
4
13
|
## 5.9.0 / 2021-01-12
|
5
14
|
### Added
|
6
15
|
* Move to keep up with Rubocop releases
|
data/README.md
CHANGED
@@ -1,4 +1,4 @@
|
|
1
|
-
## NdrDevSupport [![Maintainability](https://api.codeclimate.com/v1/badges/2b2a644964f2aa930f81/maintainability)](https://codeclimate.com/github/PublicHealthEngland/ndr_dev_support/maintainability) [![Build Status](https://
|
1
|
+
## NdrDevSupport [![Maintainability](https://api.codeclimate.com/v1/badges/2b2a644964f2aa930f81/maintainability)](https://codeclimate.com/github/PublicHealthEngland/ndr_dev_support/maintainability) [![Build Status](https://github.com/publichealthengland/ndr_dev_support/workflows/Test/badge.svg)](https://github.com/publichealthengland/ndr_dev_support/actions?query=workflow%3Atest) [![Gem Version](https://badge.fury.io/rb/ndr_dev_support.svg)](https://badge.fury.io/rb/ndr_dev_support)
|
2
2
|
|
3
3
|
This is the Public Health England (PHE) National Disease Registers (NDR) Developer Support ruby gem,
|
4
4
|
providing:
|
@@ -156,6 +156,22 @@ end
|
|
156
156
|
|
157
157
|
If tests still fail, they'll fail as normal. If tests pass after flakey failure, they'll be flagged to the RakeCI server, and rendered in purple on Slack.
|
158
158
|
|
159
|
+
#### Repeating Flakey Tests
|
160
|
+
|
161
|
+
To aid with investigations into potentially-flakey tests, `ndr_dev_support` also provides the ability to run an integration test repeatedly (by default, 100 times):
|
162
|
+
|
163
|
+
```ruby
|
164
|
+
test_repeatedly 'thing that we think might fail' do
|
165
|
+
# something flakey
|
166
|
+
end
|
167
|
+
|
168
|
+
test_repeatedly 'thing that we think might fail very occassionally', times: 1000 do
|
169
|
+
# something slightly flakey
|
170
|
+
end
|
171
|
+
```
|
172
|
+
|
173
|
+
This may be faster to work with than repeatedly executing the entire test runner in a bash loop, for example.
|
174
|
+
|
159
175
|
### Deployment support
|
160
176
|
|
161
177
|
There are various capistrano plugins in the `ndr_dev_support/capistrano` directory - see each one for details.
|
data/code_safety.yml
CHANGED
@@ -1,5 +1,17 @@
|
|
1
1
|
---
|
2
2
|
file safety:
|
3
|
+
".github/CODEOWNERS":
|
4
|
+
comments:
|
5
|
+
reviewed_by: josh.pencheon
|
6
|
+
safe_revision: ae4a81930ad3ab1b858474bc73b714aeed0f83a8
|
7
|
+
".github/workflows/lint.yml":
|
8
|
+
comments:
|
9
|
+
reviewed_by: josh.pencheon
|
10
|
+
safe_revision: 0ce43640c417e174054f903fd82043948ebe8ccb
|
11
|
+
".github/workflows/test.yml":
|
12
|
+
comments:
|
13
|
+
reviewed_by: josh.pencheon
|
14
|
+
safe_revision: 761cacc344191277f4e07ea5982fce89f7a47e0c
|
3
15
|
".gitignore":
|
4
16
|
comments:
|
5
17
|
reviewed_by: josh.pencheon
|
@@ -12,14 +24,10 @@ file safety:
|
|
12
24
|
comments:
|
13
25
|
reviewed_by: josh.pencheon
|
14
26
|
safe_revision: 6211cff0ce44645ed3752723b5b0ee65f24c66aa
|
15
|
-
".travis.yml":
|
16
|
-
comments:
|
17
|
-
reviewed_by: josh.pencheon
|
18
|
-
safe_revision: e5ac2d160975264f40bac7e871b61aa6b54f7af2
|
19
27
|
CHANGELOG.md:
|
20
28
|
comments:
|
21
29
|
reviewed_by: josh.pencheon
|
22
|
-
safe_revision:
|
30
|
+
safe_revision: fcdfdf7a55adc46a601a9cc501d36411e512015d
|
23
31
|
CODE_OF_CONDUCT.md:
|
24
32
|
comments:
|
25
33
|
reviewed_by: timgentry
|
@@ -35,7 +43,7 @@ file safety:
|
|
35
43
|
README.md:
|
36
44
|
comments:
|
37
45
|
reviewed_by: josh.pencheon
|
38
|
-
safe_revision:
|
46
|
+
safe_revision: a18f35895baebb1b0971043b3b3f666bb4eb9b5c
|
39
47
|
Rakefile:
|
40
48
|
comments:
|
41
49
|
reviewed_by: josh.pencheon
|
@@ -143,7 +151,7 @@ file safety:
|
|
143
151
|
lib/ndr_dev_support/integration_testing/flakey_tests.rb:
|
144
152
|
comments:
|
145
153
|
reviewed_by: josh.pencheon
|
146
|
-
safe_revision:
|
154
|
+
safe_revision: a18f35895baebb1b0971043b3b3f666bb4eb9b5c
|
147
155
|
lib/ndr_dev_support/rake_ci/brakeman_helper.rb:
|
148
156
|
comments:
|
149
157
|
reviewed_by: josh.pencheon
|
@@ -187,11 +195,11 @@ file safety:
|
|
187
195
|
lib/ndr_dev_support/rubocop/executor.rb:
|
188
196
|
comments:
|
189
197
|
reviewed_by: josh.pencheon
|
190
|
-
safe_revision:
|
198
|
+
safe_revision: 1d8a97dc5dd2e58bfd689d5e2c94e077a4ca0649
|
191
199
|
lib/ndr_dev_support/rubocop/inject.rb:
|
192
200
|
comments:
|
193
201
|
reviewed_by: josh.pencheon
|
194
|
-
safe_revision:
|
202
|
+
safe_revision: 1d8a97dc5dd2e58bfd689d5e2c94e077a4ca0649
|
195
203
|
lib/ndr_dev_support/rubocop/range_augmenter.rb:
|
196
204
|
comments:
|
197
205
|
reviewed_by: josh.pencheon
|
@@ -199,7 +207,7 @@ file safety:
|
|
199
207
|
lib/ndr_dev_support/rubocop/range_finder.rb:
|
200
208
|
comments:
|
201
209
|
reviewed_by: josh.pencheon
|
202
|
-
safe_revision:
|
210
|
+
safe_revision: 1d8a97dc5dd2e58bfd689d5e2c94e077a4ca0649
|
203
211
|
lib/ndr_dev_support/rubocop/reporter.rb:
|
204
212
|
comments:
|
205
213
|
reviewed_by: josh.pencheon
|
@@ -215,12 +223,12 @@ file safety:
|
|
215
223
|
lib/ndr_dev_support/version.rb:
|
216
224
|
comments:
|
217
225
|
reviewed_by: josh.pencheon
|
218
|
-
safe_revision:
|
226
|
+
safe_revision: fcdfdf7a55adc46a601a9cc501d36411e512015d
|
219
227
|
lib/tasks/audit_code.rake:
|
220
228
|
comments: Identical to the version reviewed by josh.pencheon when contained within
|
221
229
|
ndr_support
|
222
230
|
reviewed_by: josh.pencheon
|
223
|
-
safe_revision:
|
231
|
+
safe_revision: 7bd9e27c4995a9329c1e2076f6c9a772f1da48d0
|
224
232
|
lib/tasks/ci/brakeman.rake:
|
225
233
|
comments:
|
226
234
|
reviewed_by: josh.pencheon
|
@@ -288,7 +296,7 @@ file safety:
|
|
288
296
|
ndr_dev_support.gemspec:
|
289
297
|
comments:
|
290
298
|
reviewed_by: josh.pencheon
|
291
|
-
safe_revision:
|
299
|
+
safe_revision: 8d5bef0365849bd87cd3239a722fc21df1e55c48
|
292
300
|
test/daemon/ci_server_test.rb:
|
293
301
|
comments:
|
294
302
|
reviewed_by: josh.pencheon
|
@@ -15,6 +15,12 @@ module NdrDevSupport
|
|
15
15
|
self.attempts_per_test = attempts_per_test.merge(test_name.to_s => attempts)
|
16
16
|
end
|
17
17
|
end
|
18
|
+
|
19
|
+
def test_repeatedly(description, times: 100, &block)
|
20
|
+
(1..times).map do |n|
|
21
|
+
test("#{description} - #{n}/#{times}", &block)
|
22
|
+
end
|
23
|
+
end
|
18
24
|
end
|
19
25
|
|
20
26
|
def flakes
|
@@ -10,7 +10,7 @@ module NdrDevSupport
|
|
10
10
|
class << self
|
11
11
|
# Use RuboCop to produce a list of all files that should be scanned.
|
12
12
|
def target_files
|
13
|
-
@target_files ||= `rubocop -L`.each_line.map(&:strip)
|
13
|
+
@target_files ||= `rubocop -L 2>/dev/null`.each_line.map(&:strip)
|
14
14
|
end
|
15
15
|
end
|
16
16
|
|
@@ -23,7 +23,7 @@ module NdrDevSupport
|
|
23
23
|
def offenses_by_file
|
24
24
|
return [] if @filenames.empty?
|
25
25
|
|
26
|
-
output = JSON.parse(`rubocop --format json #{escaped_paths.join(' ')}`)
|
26
|
+
output = JSON.parse(`rubocop --format json #{escaped_paths.join(' ')} 2>/dev/null`)
|
27
27
|
|
28
28
|
output['files'].each_with_object({}) do |file_output, result|
|
29
29
|
result[file_output['path']] = file_output['offenses']
|
data/lib/tasks/audit_code.rake
CHANGED
@@ -1,3 +1,4 @@
|
|
1
|
+
require 'csv'
|
1
2
|
require 'pathname'
|
2
3
|
require 'yaml'
|
3
4
|
|
@@ -10,9 +11,6 @@ SAFETY_FILE =
|
|
10
11
|
Pathname.new('code_safety.yml').expand_path
|
11
12
|
end
|
12
13
|
|
13
|
-
# Temporary overrides to only audit external access files
|
14
|
-
SAFETY_REPOS = [['/svn/era', '/svn/extra/era/external-access']]
|
15
|
-
|
16
14
|
# Returns the (Bundler aware) rake command
|
17
15
|
def rake_cmd
|
18
16
|
ENV['BUNDLE_BIN_PATH'] ? 'bundle exec rake' : 'rake'
|
@@ -48,12 +46,11 @@ end
|
|
48
46
|
|
49
47
|
# Parameter max_print is number of entries to print before truncating output
|
50
48
|
# (negative value => print all)
|
51
|
-
def audit_code_safety(max_print = 20, ignore_new = false, show_diffs = false,
|
52
|
-
puts 'Running source code safety audit script.'
|
53
|
-
puts
|
54
|
-
|
49
|
+
def audit_code_safety(max_print = 20, ignore_new = false, show_diffs = false, usr = 'usr', interactive = false, outfile = nil, filter = nil)
|
55
50
|
max_print = 1_000_000 if max_print.negative?
|
56
51
|
show_diffs = true if interactive
|
52
|
+
ignore_new = true if filter
|
53
|
+
|
57
54
|
file_safety = load_file_safety
|
58
55
|
file_safety.each_value do |v|
|
59
56
|
rev = v['safe_revision']
|
@@ -63,76 +60,143 @@ def audit_code_safety(max_print = 20, ignore_new = false, show_diffs = false, sh
|
|
63
60
|
|
64
61
|
safety_repo = trunk_repo = get_trunk_repo
|
65
62
|
|
66
|
-
# TODO: below is broken for git-svn
|
67
|
-
# Is it needed?
|
68
|
-
|
69
|
-
SAFETY_REPOS.each do |suffix, alt|
|
70
|
-
# Temporarily override to only audit a different file list
|
71
|
-
if safety_repo.end_with?(suffix)
|
72
|
-
safety_repo = safety_repo[0...-suffix.length] + alt
|
73
|
-
break
|
74
|
-
end
|
75
|
-
end
|
76
|
-
|
77
63
|
if ignore_new
|
78
64
|
puts "Not checking for new files in #{safety_repo}"
|
79
65
|
else
|
80
66
|
puts "Checking for new files in #{safety_repo}"
|
81
|
-
|
82
|
-
# Ignore subdirectories, and exclude code_safety.yml by default.
|
83
|
-
new_files.delete_if { |f| f =~ /[\/\\]$/ || Pathname.new(f).expand_path == SAFETY_FILE }
|
84
|
-
new_files.each { |f| add_new_file_to_file_safety(file_safety, f) }
|
85
|
-
update_safety_file(file_safety) # Save changes before checking latest revisions
|
67
|
+
add_new_files(safety_repo, file_safety)
|
86
68
|
end
|
87
|
-
puts "Updating latest revisions for #{file_safety.size} files"
|
88
|
-
set_last_changed_revision(trunk_repo, file_safety, file_safety.keys)
|
89
|
-
puts "\nSummary:"
|
90
|
-
puts "Number of files originally in #{SAFETY_FILE}: #{orig_count}"
|
91
|
-
puts "Number of new files added: #{file_safety.size - orig_count}"
|
92
69
|
|
93
70
|
missing_files = file_safety.keys.reject { |path| File.file?(path) }
|
71
|
+
abort(<<~MESSAGE) if missing_files.any?
|
94
72
|
|
95
|
-
|
96
|
-
|
97
|
-
|
98
|
-
|
73
|
+
The following file(s) no longer exist in the repository:
|
74
|
+
#{missing_files.join("\n ")}
|
75
|
+
|
76
|
+
Please run `#{rake_cmd} audit:tidy_code_safety_file` to remove them.
|
77
|
+
MESSAGE
|
78
|
+
|
79
|
+
if filter
|
80
|
+
filter_file_safety(file_safety, filter)
|
81
|
+
puts "Applying provided file filter: #{file_safety.size} file(s) remain"
|
99
82
|
end
|
100
83
|
|
101
|
-
|
102
|
-
|
103
|
-
|
104
|
-
|
105
|
-
|
106
|
-
|
107
|
-
|
84
|
+
if interactive
|
85
|
+
run_review_wizard(trunk_repo, file_safety, usr)
|
86
|
+
|
87
|
+
# Post-wizard, reload the results and continue to print a summary:
|
88
|
+
file_safety = load_file_safety
|
89
|
+
filter_file_safety(file_safety, filter) if filter
|
90
|
+
|
91
|
+
max_print = 0
|
92
|
+
show_diffs = false
|
93
|
+
else
|
94
|
+
# Get updates for all files in one go:
|
95
|
+
puts "Updating latest revisions for #{file_safety.size} files"
|
96
|
+
set_last_changed_revisions(trunk_repo, file_safety)
|
108
97
|
end
|
109
|
-
|
110
|
-
|
98
|
+
|
99
|
+
print_summary(file_safety, usr, trunk_repo, max_print, show_diffs, orig_count, ignore_new, outfile, filter)
|
100
|
+
end
|
101
|
+
|
102
|
+
def filter_file_safety(file_safety, filter)
|
103
|
+
filtered_paths = CSV.read(filter, headers: true).map { |row| row.to_h.fetch('path') }
|
104
|
+
file_safety.select! { |fname, _entry| filtered_paths.include?(fname) }
|
105
|
+
end
|
106
|
+
|
107
|
+
def run_review_wizard(trunk_repo, file_safety, usr)
|
108
|
+
puts <<~INTRO
|
109
|
+
|
110
|
+
In interactive mode, you'll be presented with one diff at a time for review.
|
111
|
+
|
112
|
+
For each diff, you can enter a decision (if you can make one) along with
|
113
|
+
any comments you may have, before being taken to the next diff.
|
114
|
+
|
115
|
+
INTRO
|
116
|
+
|
117
|
+
file_safety.each_key do |fname|
|
118
|
+
set_last_changed_revision(trunk_repo, file_safety, fname)
|
119
|
+
next unless print_file_safety(file_safety, fname, false, true)
|
120
|
+
|
121
|
+
print_file_diffs(file_safety, trunk_repo, fname, usr, true)
|
122
|
+
end
|
123
|
+
end
|
124
|
+
|
125
|
+
def print_summary(file_safety, usr, trunk_repo, max_print, show_diffs, orig_count, ignore_new, outfile, filter)
|
126
|
+
puts "\nSummary:"
|
127
|
+
puts "Number of files originally in #{SAFETY_FILE}: #{orig_count}"
|
128
|
+
puts "Number of new files added: #{file_safety.size - orig_count}" unless ignore_new
|
129
|
+
|
130
|
+
# Now generate statistics:
|
131
|
+
unknown = file_safety.values.select { |x| unreviewed?(x) }
|
132
|
+
unsafe = file_safety.values.select { |x| stale_review?(x) }
|
133
|
+
puts "Number of#{' filtered' if filter} files with no safe version: #{unknown.size}"
|
134
|
+
puts "Number of#{' filtered' if filter} files which are no longer safe: #{unsafe.size}"
|
111
135
|
puts
|
112
136
|
printed = []
|
113
|
-
|
137
|
+
|
114
138
|
file_list =
|
115
|
-
|
116
|
-
|
117
|
-
|
118
|
-
file_safety.keys.sort
|
119
|
-
end
|
139
|
+
file_safety.
|
140
|
+
sort_by { |_k, v| v.nil? ? -100 : v['last_changed_rev'].to_i }.
|
141
|
+
map(&:first)
|
120
142
|
|
121
143
|
file_list.each do |f|
|
122
144
|
printed << f if print_file_safety(file_safety, f, false, printed.size >= max_print)
|
123
145
|
end
|
124
|
-
puts "... and #{printed.size - max_print} others" if printed.size > max_print
|
146
|
+
puts "... and #{printed.size - max_print} others" if printed.size > max_print && max_print > 0
|
125
147
|
if show_diffs
|
126
148
|
puts
|
127
149
|
printed.each do |f|
|
128
|
-
print_file_diffs(file_safety, trunk_repo, f, usr,
|
150
|
+
print_file_diffs(file_safety, trunk_repo, f, usr, false)
|
129
151
|
end
|
130
152
|
end
|
131
153
|
|
154
|
+
export_csv(trunk_repo, file_safety, printed, outfile) if outfile
|
155
|
+
|
132
156
|
# Returns `true` unless there are pending reviews:
|
133
157
|
unsafe.length.zero? && unknown.length.zero?
|
134
158
|
end
|
135
159
|
|
160
|
+
def export_csv(repo, file_safety, printed, outfile)
|
161
|
+
CSV.open(outfile, 'w') do |csv|
|
162
|
+
csv << %w[path last_changed_revision safe_revision diff_lines]
|
163
|
+
|
164
|
+
file_safety.each do |fname, entry|
|
165
|
+
# Only emit files needing review:
|
166
|
+
next unless printed.include?(fname)
|
167
|
+
|
168
|
+
last_changed_revision = entry['last_changed_rev']
|
169
|
+
safe_revision = entry['safe_revision']
|
170
|
+
|
171
|
+
_cmd, diffs = capture_file_diffs(repo, fname, safe_revision, last_changed_revision)
|
172
|
+
diff_lines = diffs.split("\n").length
|
173
|
+
|
174
|
+
csv << [fname, last_changed_revision, safe_revision, diff_lines]
|
175
|
+
end
|
176
|
+
end
|
177
|
+
|
178
|
+
puts <<~MESSAGE
|
179
|
+
|
180
|
+
CSV output exported to: #{outfile}
|
181
|
+
|
182
|
+
MESSAGE
|
183
|
+
end
|
184
|
+
|
185
|
+
def stale_review?(entry)
|
186
|
+
return false if unreviewed?(entry)
|
187
|
+
|
188
|
+
# Could be comparing Git SHAs, or SVN revisions:
|
189
|
+
entry['last_changed_rev'] != entry['safe_revision']
|
190
|
+
end
|
191
|
+
|
192
|
+
def unreviewed?(entry)
|
193
|
+
entry['safe_revision'].nil?
|
194
|
+
end
|
195
|
+
|
196
|
+
def needs_review?(entry)
|
197
|
+
unreviewed?(entry) || stale_review?(entry)
|
198
|
+
end
|
199
|
+
|
136
200
|
# Print summary details of a file's known safety
|
137
201
|
# If not verbose, only prints details for unsafe files
|
138
202
|
# Returns true if anything printed (or would have been printed if silent),
|
@@ -210,14 +274,11 @@ def get_trunk_repo
|
|
210
274
|
case repository_type
|
211
275
|
when 'svn'
|
212
276
|
repo_info = %x[svn info]
|
213
|
-
puts 'svn case'
|
214
277
|
repo_info.split("\n").select { |x| x =~ /^URL: / }.collect { |x| x[5..-1] }.first
|
215
278
|
when 'git-svn'
|
216
|
-
puts 'git-svn case'
|
217
279
|
repo_info = %x[git svn info]
|
218
280
|
repo_info.split("\n").select { |x| x =~ /^URL: / }.collect { |x| x[5..-1] }.first
|
219
281
|
when 'git'
|
220
|
-
puts 'git case'
|
221
282
|
repo_info = %x[git remote -v]
|
222
283
|
repo_info.split("\n").first[7..-9]
|
223
284
|
else
|
@@ -225,69 +286,45 @@ def get_trunk_repo
|
|
225
286
|
end
|
226
287
|
end
|
227
288
|
|
228
|
-
def
|
229
|
-
|
230
|
-
|
231
|
-
|
232
|
-
|
233
|
-
|
234
|
-
|
289
|
+
def add_new_files(safety_repo, file_safety)
|
290
|
+
new_files =
|
291
|
+
case repository_type
|
292
|
+
when 'svn', 'git-svn'
|
293
|
+
%x[svn ls -R "#{safety_repo}"].split("\n")
|
294
|
+
when 'git'
|
295
|
+
%x[git ls-files].split("\n")
|
296
|
+
else
|
297
|
+
[]
|
298
|
+
end
|
235
299
|
|
236
|
-
|
237
|
-
|
238
|
-
|
239
|
-
|
240
|
-
|
241
|
-
|
242
|
-
end
|
300
|
+
# Ignore subdirectories, and exclude code_safety.yml by default.
|
301
|
+
new_files.delete_if { |f| f =~ /[\/\\]$/ || Pathname.new(f).expand_path == SAFETY_FILE }
|
302
|
+
|
303
|
+
# Save changes before checking latest revisions
|
304
|
+
new_files.each { |f| add_new_file_to_file_safety(file_safety, f) }
|
305
|
+
update_safety_file(file_safety)
|
243
306
|
end
|
244
307
|
|
245
308
|
# Fill in the latest changed revisions in a file safety map.
|
246
309
|
# (Don't write this data to the YAML file, as it is intrinsic to the SVN
|
247
310
|
# repository.)
|
248
|
-
def
|
249
|
-
|
250
|
-
|
251
|
-
|
252
|
-
|
253
|
-
|
254
|
-
|
255
|
-
info = %x[git log -n 1 -- #{f}].split("\n").first[7..-1]
|
256
|
-
if info.nil? || info.empty?
|
257
|
-
file_safety[f]['last_changed_rev'] = -1
|
258
|
-
else
|
259
|
-
file_safety[f]['last_changed_rev'] = info
|
260
|
-
end
|
261
|
-
# Show progress
|
262
|
-
print '.' if (i % dot_freq) == 0
|
263
|
-
end
|
264
|
-
puts
|
265
|
-
when 'git-svn', 'svn'
|
266
|
-
fnames = file_safety.keys if fnames.nil?
|
267
|
-
|
268
|
-
fnames.each_with_index do |f, i|
|
269
|
-
last_revision = get_last_changed_revision(repo, f)
|
270
|
-
if last_revision.nil? || last_revision.empty?
|
271
|
-
file_safety[f]['last_changed_rev'] = -1
|
272
|
-
else
|
273
|
-
file_safety[f]['last_changed_rev'] = last_revision
|
274
|
-
end
|
275
|
-
# Show progress
|
276
|
-
print '.' if (i % dot_freq) == 0
|
277
|
-
end
|
278
|
-
puts
|
279
|
-
# NOTE: Do we need the following for retries?
|
280
|
-
# if retries && result.size != fnames.size && fnames.size > 1
|
281
|
-
# # At least one invalid (deleted file --> subsequent arguments ignored)
|
282
|
-
# # Try each file individually
|
283
|
-
# # (It would probably be safe to continue from the extra_info.size argument)
|
284
|
-
# puts "Retrying (got #{result.size}, expected #{fnames.size})" if debug >= 2
|
285
|
-
# result = []
|
286
|
-
# fnames.each{ |f|
|
287
|
-
# result += svn_info_entries([f], repo, false, debug)
|
288
|
-
# }
|
289
|
-
# end
|
311
|
+
def set_last_changed_revisions(repo, file_safety)
|
312
|
+
fnames = file_safety.keys
|
313
|
+
dot_freq = (fnames.size / 40.0).ceil # Print up to 40 progress dots
|
314
|
+
|
315
|
+
fnames.each_with_index do |f, i|
|
316
|
+
set_last_changed_revision(repo, file_safety, f)
|
317
|
+
print '.' if (i % dot_freq) == 0
|
290
318
|
end
|
319
|
+
puts
|
320
|
+
end
|
321
|
+
|
322
|
+
# Fill in the latest changed revision for the given file in a file safety map.
|
323
|
+
def set_last_changed_revision(repo, file_safety, fname)
|
324
|
+
last_revision = get_last_changed_revision(repo, fname)
|
325
|
+
last_revision = -1 if last_revision.blank?
|
326
|
+
|
327
|
+
file_safety[fname]['last_changed_rev'] = last_revision
|
291
328
|
end
|
292
329
|
|
293
330
|
# Return the last changed revision
|
@@ -331,31 +368,44 @@ def root_revision
|
|
331
368
|
end
|
332
369
|
end
|
333
370
|
|
371
|
+
def capture_file_diffs(repo, fname, safe_revision, repolatest)
|
372
|
+
cmd =
|
373
|
+
case repository_type
|
374
|
+
when 'git'
|
375
|
+
cmd = ['git', '--no-pager', 'diff', '--color', '-b', "#{safe_revision}..#{repolatest}", fname]
|
376
|
+
when 'git-svn', 'svn'
|
377
|
+
cmd = ['svn', 'diff', '-r', "#{safe_revision.to_i}:#{repolatest.to_i}", '-x', '-b', "#{repo}/#{fname}"]
|
378
|
+
end
|
379
|
+
|
380
|
+
stdout_and_err_str, _status = Open3.capture2e(*cmd)
|
381
|
+
|
382
|
+
if stdout_and_err_str.start_with?('fatal: Invalid revision range ')
|
383
|
+
abort <<~ERROR
|
384
|
+
Error running:
|
385
|
+
#{cmd}
|
386
|
+
|
387
|
+
Invalid commit ID in code_safety.yml for #{fname}? (#{safe_revision})
|
388
|
+
ERROR
|
389
|
+
end
|
390
|
+
|
391
|
+
[cmd.join(' '), stdout_and_err_str]
|
392
|
+
end
|
393
|
+
|
334
394
|
def print_repo_file_diffs(repolatest, repo, fname, usr, safe_revision, interactive)
|
335
395
|
require 'open3'
|
336
396
|
require 'highline/import'
|
337
397
|
|
338
398
|
if interactive
|
339
|
-
ask("
|
399
|
+
ask("<%= color('Press Enter to continue ...', :yellow) %>")
|
340
400
|
system('clear')
|
341
401
|
system("printf '\033[3J'") # clear the scrollback
|
342
402
|
end
|
343
403
|
|
344
|
-
cmd =
|
345
|
-
|
346
|
-
|
347
|
-
|
348
|
-
|
349
|
-
cmd = ['svn', 'diff', '-r', "#{safe_revision.to_i}:#{repolatest.to_i}", '-x', '-b', "#{repo}/#{fname}"]
|
350
|
-
end
|
351
|
-
if cmd
|
352
|
-
puts(cmd.join(' '))
|
353
|
-
stdout_and_err_str, _status = Open3.capture2e(*cmd)
|
354
|
-
puts 'Invalid commit ID in code_safety.yml ' + safe_revision if stdout_and_err_str.start_with?('fatal: Invalid revision range ')
|
355
|
-
puts(stdout_and_err_str)
|
356
|
-
else
|
357
|
-
puts 'Unknown repo'
|
358
|
-
end
|
404
|
+
cmd, diffs = capture_file_diffs(repo, fname, safe_revision, repolatest)
|
405
|
+
puts cmd
|
406
|
+
puts
|
407
|
+
puts diffs
|
408
|
+
puts
|
359
409
|
|
360
410
|
if interactive
|
361
411
|
response = ask("Flag #{fname} changes safe? [Yes|No|Abort]: ") { |q| q.case = :down }
|
@@ -363,18 +413,19 @@ def print_repo_file_diffs(repolatest, repo, fname, usr, safe_revision, interacti
|
|
363
413
|
puts 'Flagging as safe...'
|
364
414
|
release = get_release(repolatest)
|
365
415
|
if usr.to_s.strip.empty?
|
366
|
-
usr = ask('File reviewed by:') do |q|
|
416
|
+
usr = ask('File reviewed by: ') do |q|
|
367
417
|
q.whitespace = :strip_and_collapse
|
368
|
-
q.validate = /\A[\w \-.]+\
|
418
|
+
q.validate = /\A[\w \-.]+\z/
|
369
419
|
end
|
370
420
|
end
|
371
|
-
comment = ask('Please write your comments (optional):')
|
421
|
+
comment = ask('Please write your comments (optional): ')
|
372
422
|
# use to_s to convert response from !ruby/string:HighLine::String to String
|
373
423
|
flag_file_as_safe(release, usr.to_s, comment.to_s, fname)
|
374
|
-
elsif
|
375
|
-
|
424
|
+
elsif response =~ /\Aa/i
|
425
|
+
say("\n<%= color('Aborted by user.', :red) %>")
|
426
|
+
abort
|
376
427
|
else
|
377
|
-
say("\n<%= color('
|
428
|
+
say("\n<%= color('Skipping review of #{fname}.', :magenta) %>")
|
378
429
|
end
|
379
430
|
else
|
380
431
|
puts 'To flag the changes to this file as safe, run:'
|
@@ -436,17 +487,35 @@ File #{SAFETY_FILE} lists the safety and revision information
|
|
436
487
|
of the era source code. This task updates the list, and [TODO] warns about
|
437
488
|
files which have changed since they were last verified as safe."
|
438
489
|
task(:code) do
|
439
|
-
puts
|
440
|
-
|
490
|
+
puts <<~USAGE
|
491
|
+
|
492
|
+
NDR's code auditing tool.
|
493
|
+
|
494
|
+
Usage:
|
495
|
+
|
496
|
+
#{rake_cmd} audit:code
|
497
|
+
[interactive=false|true] # use an interactive wizard to perform code review
|
498
|
+
[max_print=n] # limit the number of summary rows to n
|
499
|
+
[ignore_new=false|true] # don't add new files to code_safety.yml
|
500
|
+
[show_diffs=false|true] # display the full diff for each file
|
501
|
+
[reviewed_by=usr] # your name, to author to any reviews added
|
502
|
+
[filter=path/to/input.csv] # filter reviewing to only files specified in the filter file
|
503
|
+
[outfile=path/to/output.csv] # dump a summary of outstanding review to CSV
|
504
|
+
|
505
|
+
USAGE
|
506
|
+
|
507
|
+
puts "This is a #{repository_type} repository."
|
508
|
+
puts
|
441
509
|
|
442
510
|
ignore_new = (ENV['ignore_new'].to_s =~ /\Atrue\Z/i)
|
443
511
|
show_diffs = (ENV['show_diffs'].to_s =~ /\Atrue\Z/i)
|
444
|
-
show_in_priority = (ENV['show_in_priority'].to_s =~ /\Atrue\Z/i)
|
445
512
|
max_print = ENV['max_print'] =~ /\A-?[0-9][0-9]*\Z/ ? ENV['max_print'].to_i : 20
|
446
513
|
reviewer = ENV['reviewed_by']
|
447
514
|
interactive = (ENV['interactive'].to_s =~ /\Atrue\Z/i)
|
515
|
+
filter = ENV['filter']
|
516
|
+
outfile = ENV['outfile']
|
448
517
|
|
449
|
-
all_safe = audit_code_safety(max_print, ignore_new, show_diffs,
|
518
|
+
all_safe = audit_code_safety(max_print, ignore_new, show_diffs, reviewer, interactive, outfile, filter)
|
450
519
|
|
451
520
|
unless show_diffs || interactive
|
452
521
|
puts "To show file diffs, run: #{rake_cmd} audit:code max_print=-1 show_diffs=true"
|
data/ndr_dev_support.gemspec
CHANGED
@@ -12,9 +12,7 @@ Gem::Specification.new do |spec|
|
|
12
12
|
spec.homepage = 'https://github.com/PublicHealthEngland/ndr_dev_support'
|
13
13
|
spec.license = 'MIT'
|
14
14
|
|
15
|
-
spec.files = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
|
16
|
-
# SECURE BNS 2018-08-06: Minimise sharing of (public-key encrypted) slack secrets in .travis.yml
|
17
|
-
spec.files -= %w[.travis.yml] # Not needed in the gem
|
15
|
+
spec.files = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(\.github|test|spec|features)/}) }
|
18
16
|
spec.bindir = 'exe'
|
19
17
|
spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
|
20
18
|
spec.require_paths = ['lib']
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: ndr_dev_support
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 5.
|
4
|
+
version: 5.10.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- NCRS Development Team
|
8
8
|
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date: 2021-01-
|
11
|
+
date: 2021-01-29 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: pry
|