cassette_guard 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/CHANGELOG.md +15 -0
- data/LICENSE.txt +21 -0
- data/README.md +454 -0
- data/exe/cassette_guard +6 -0
- data/lib/cassette_guard/baseline.rb +59 -0
- data/lib/cassette_guard/body.rb +100 -0
- data/lib/cassette_guard/cassette.rb +81 -0
- data/lib/cassette_guard/cli.rb +309 -0
- data/lib/cassette_guard/config.rb +106 -0
- data/lib/cassette_guard/detectors/age.rb +46 -0
- data/lib/cassette_guard/detectors/pii.rb +45 -0
- data/lib/cassette_guard/detectors/secret.rb +83 -0
- data/lib/cassette_guard/detectors/size.rb +28 -0
- data/lib/cassette_guard/detectors/usage.rb +45 -0
- data/lib/cassette_guard/encodings.rb +79 -0
- data/lib/cassette_guard/fake_value.rb +103 -0
- data/lib/cassette_guard/fields.rb +74 -0
- data/lib/cassette_guard/finding.rb +33 -0
- data/lib/cassette_guard/finding_group.rb +37 -0
- data/lib/cassette_guard/format.rb +37 -0
- data/lib/cassette_guard/headers.rb +33 -0
- data/lib/cassette_guard/line_locator.rb +37 -0
- data/lib/cassette_guard/py_yaml.rb +228 -0
- data/lib/cassette_guard/reporters/json.rb +24 -0
- data/lib/cassette_guard/reporters/sarif.rb +67 -0
- data/lib/cassette_guard/reporters/text.rb +76 -0
- data/lib/cassette_guard/rules.rb +127 -0
- data/lib/cassette_guard/scanner.rb +131 -0
- data/lib/cassette_guard/scrubber.rb +138 -0
- data/lib/cassette_guard/trimmer.rb +70 -0
- data/lib/cassette_guard/usage.rb +47 -0
- data/lib/cassette_guard/version.rb +5 -0
- data/lib/cassette_guard.rb +40 -0
- metadata +84 -0
|
@@ -0,0 +1,81 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module CassetteGuard
|
|
4
|
+
# A cassette recorded by Ruby's VCR (`http_interactions`) or Python's vcrpy (`interactions`).
|
|
5
|
+
class Cassette
|
|
6
|
+
PERMITTED_CLASSES = [Symbol, Time, Date].freeze
|
|
7
|
+
RECORDERS = { "http_interactions" => :vcr, "interactions" => :vcrpy }.freeze
|
|
8
|
+
|
|
9
|
+
attr_reader :path, :data, :format, :raw, :unsupported_tags
|
|
10
|
+
|
|
11
|
+
def self.load(path)
|
|
12
|
+
raw = File.read(path, encoding: Encoding::UTF_8)
|
|
13
|
+
format = File.extname(path).casecmp?(".json") ? :json : :yaml
|
|
14
|
+
return new(path:, raw:, data: JSON.parse(raw), format:) if format == :json
|
|
15
|
+
|
|
16
|
+
document = Psych.parse(raw)
|
|
17
|
+
root = document ? document.root : nil
|
|
18
|
+
return new(path:, raw:, data: YAML.safe_load(raw, permitted_classes: PERMITTED_CLASSES, aliases: true), format:) unless PyYaml.vcrpy?(root)
|
|
19
|
+
|
|
20
|
+
data, unsupported_tags = PyYaml.load(root)
|
|
21
|
+
new(path:, raw:, data:, format:, unsupported_tags:)
|
|
22
|
+
rescue Psych::Exception, JSON::ParserError, ArgumentError, KeyError => e
|
|
23
|
+
raise Error, "cannot parse (#{e.message.lines.first.to_s.strip})"
|
|
24
|
+
end
|
|
25
|
+
|
|
26
|
+
def initialize(path:, raw:, data:, format:, unsupported_tags: [])
|
|
27
|
+
@path = path
|
|
28
|
+
@raw = raw
|
|
29
|
+
@data = data
|
|
30
|
+
@format = format
|
|
31
|
+
@unsupported_tags = unsupported_tags
|
|
32
|
+
end
|
|
33
|
+
|
|
34
|
+
def recorder
|
|
35
|
+
return nil unless data.is_a?(Hash)
|
|
36
|
+
|
|
37
|
+
RECORDERS.find { |key, _| data[key].is_a?(Array) }&.last
|
|
38
|
+
end
|
|
39
|
+
|
|
40
|
+
def vcr?
|
|
41
|
+
!recorder.nil?
|
|
42
|
+
end
|
|
43
|
+
|
|
44
|
+
def interactions
|
|
45
|
+
key = RECORDERS.key(recorder)
|
|
46
|
+
key ? data[key].grep(Hash) : []
|
|
47
|
+
end
|
|
48
|
+
|
|
49
|
+
# Older vcrpy cassettes can hold `!!python/...` tags that only vcrpy's loader rebuilds.
|
|
50
|
+
def writable?
|
|
51
|
+
unsupported_tags.empty?
|
|
52
|
+
end
|
|
53
|
+
|
|
54
|
+
def unwritable_reason
|
|
55
|
+
tags = unsupported_tags.map { |tag| tag.sub(PyYaml::TAG, "!!") }.join(", ")
|
|
56
|
+
"holds YAML tags only vcrpy can rebuild (#{tags}); re-record it with a current vcrpy first"
|
|
57
|
+
end
|
|
58
|
+
|
|
59
|
+
# Serialized the way the recorder writes cassettes, so untouched parts produce no diff.
|
|
60
|
+
def dump
|
|
61
|
+
case [recorder, format]
|
|
62
|
+
in [:vcrpy, :yaml] then PyYaml.dump(data)
|
|
63
|
+
in [:vcrpy, :json] then "#{Encodings.python_json(JSON.pretty_generate(data, indent: " ", ascii_only: true))}\n"
|
|
64
|
+
in [_, :yaml] then YAML.dump(data)
|
|
65
|
+
else raw.include?("\n") ? JSON.pretty_generate(data) : JSON.generate(data)
|
|
66
|
+
end
|
|
67
|
+
end
|
|
68
|
+
|
|
69
|
+
def changed?
|
|
70
|
+
dump != raw
|
|
71
|
+
end
|
|
72
|
+
|
|
73
|
+
def save!
|
|
74
|
+
raise Error, unwritable_reason unless writable?
|
|
75
|
+
|
|
76
|
+
text = dump
|
|
77
|
+
File.write(path, text)
|
|
78
|
+
@raw = text
|
|
79
|
+
end
|
|
80
|
+
end
|
|
81
|
+
end
|
|
@@ -0,0 +1,309 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "optparse"
|
|
4
|
+
|
|
5
|
+
module CassetteGuard
|
|
6
|
+
class CLI
|
|
7
|
+
COMMANDS = %w[audit check scrub trim baseline version help].freeze
|
|
8
|
+
REPORTERS = { "text" => Reporters::Text, "json" => Reporters::Json, "sarif" => Reporters::Sarif }.freeze
|
|
9
|
+
|
|
10
|
+
BANNER = <<~TEXT
|
|
11
|
+
Usage: cassette_guard [COMMAND] [options] [PATH...]
|
|
12
|
+
|
|
13
|
+
Commands:
|
|
14
|
+
audit Report secrets, personal data, oversized, old and unused cassettes (default)
|
|
15
|
+
check Report only findings missing from the baseline; exit 1 if there are any (for CI)
|
|
16
|
+
scrub Replace secrets and personal data with deterministic fakes
|
|
17
|
+
trim Replace large binary bodies and base64 fields with short stubs
|
|
18
|
+
baseline Accept every current finding, so `check` only fails on new ones
|
|
19
|
+
version Print the version
|
|
20
|
+
|
|
21
|
+
PATH defaults to `cassettes:` in .cassette_guard.yml, then to the usual VCR directories.
|
|
22
|
+
TEXT
|
|
23
|
+
|
|
24
|
+
def initialize(argv, out: $stdout, err: $stderr, root: Dir.pwd)
|
|
25
|
+
@argv = argv.dup
|
|
26
|
+
@out = out
|
|
27
|
+
@err = err
|
|
28
|
+
@root = root
|
|
29
|
+
end
|
|
30
|
+
|
|
31
|
+
def run
|
|
32
|
+
command = COMMANDS.include?(@argv.first) ? @argv.shift : "audit"
|
|
33
|
+
options = parse(@argv)
|
|
34
|
+
return help if command == "help" || options[:help]
|
|
35
|
+
return version if command == "version"
|
|
36
|
+
|
|
37
|
+
@config = load_config(options)
|
|
38
|
+
files = @config.cassette_files
|
|
39
|
+
raise Error, "no cassettes found. Pass a directory, or set `cassettes:` in #{Config::FILE}." if files.empty?
|
|
40
|
+
|
|
41
|
+
public_send("run_#{command}", files, options)
|
|
42
|
+
rescue OptionParser::ParseError, Error => e
|
|
43
|
+
@err.puts("cassette_guard: #{e.message}")
|
|
44
|
+
2
|
|
45
|
+
end
|
|
46
|
+
|
|
47
|
+
def run_audit(files, options)
|
|
48
|
+
report = Scanner.new(@config).scan(files)
|
|
49
|
+
baseline = Baseline.load(@config.baseline_path)
|
|
50
|
+
shown = options[:all] ? report.findings : report.findings.reject { |finding| baseline.include?(finding) }
|
|
51
|
+
emit(render(report, shown, options), options)
|
|
52
|
+
0
|
|
53
|
+
end
|
|
54
|
+
|
|
55
|
+
def run_check(files, options)
|
|
56
|
+
report = Scanner.new(@config).scan(files)
|
|
57
|
+
baseline = Baseline.load(@config.baseline_path)
|
|
58
|
+
fresh = report.findings.reject { |finding| baseline.include?(finding) }
|
|
59
|
+
emit(render(report, fresh, options), options)
|
|
60
|
+
return 0 if fresh.empty?
|
|
61
|
+
|
|
62
|
+
@err.puts("#{Format.plural(FindingGroup.from(fresh).size, "new finding")}. Fix them, run `cassette_guard scrub`, " \
|
|
63
|
+
"or accept them with `cassette_guard baseline`.")
|
|
64
|
+
1
|
|
65
|
+
end
|
|
66
|
+
|
|
67
|
+
def run_baseline(files, _options)
|
|
68
|
+
report = Scanner.new(@config).scan(files)
|
|
69
|
+
previous = Baseline.load(@config.baseline_path)
|
|
70
|
+
baseline = Baseline.new(@config.baseline_path)
|
|
71
|
+
report.findings.each { |finding| baseline.add(finding, reason: previous.reason_for(finding) || "accepted") }
|
|
72
|
+
baseline.save!
|
|
73
|
+
@out.puts("Wrote #{Format.plural(baseline.size, "finding")} to #{@config.relative(baseline.path)} " \
|
|
74
|
+
"(#{baseline.count("accepted")} accepted, #{baseline.count("scrubbed")} scrubbed).")
|
|
75
|
+
0
|
|
76
|
+
end
|
|
77
|
+
|
|
78
|
+
def run_scrub(files, options)
|
|
79
|
+
ScrubCommand.new(@config, @out, options).call(files)
|
|
80
|
+
end
|
|
81
|
+
|
|
82
|
+
def run_trim(files, options)
|
|
83
|
+
trimmer = Trimmer.new(min_size: @config.trim_min_size)
|
|
84
|
+
total = []
|
|
85
|
+
files.each do |path|
|
|
86
|
+
cassette = Cassette.load(path)
|
|
87
|
+
next unless cassette.vcr?
|
|
88
|
+
raise Error, cassette.unwritable_reason unless cassette.writable?
|
|
89
|
+
|
|
90
|
+
trimmed = trimmer.trim(cassette)
|
|
91
|
+
next if trimmed.empty?
|
|
92
|
+
|
|
93
|
+
options[:dry_run] ? cassette.dump : cassette.save!
|
|
94
|
+
@out.puts(@config.relative(path))
|
|
95
|
+
trimmed.each do |item|
|
|
96
|
+
@out.puts(" ##{item.interaction} #{item.location} #{Format.bytes(item.bytes)} #{item.content_type}".rstrip)
|
|
97
|
+
end
|
|
98
|
+
total.concat(trimmed)
|
|
99
|
+
rescue Error => e
|
|
100
|
+
@err.puts("Skipped #{@config.relative(path)}: #{e.message}")
|
|
101
|
+
end
|
|
102
|
+
verb = options[:dry_run] ? "Would trim" : "Trimmed"
|
|
103
|
+
@out.puts("#{verb} #{Format.plural(total.size, "body")}, #{Format.bytes(total.sum(&:bytes))}.")
|
|
104
|
+
@out.puts("Run your test suite: code that reads these bodies now gets a short stub.") if total.any?
|
|
105
|
+
0
|
|
106
|
+
end
|
|
107
|
+
|
|
108
|
+
private
|
|
109
|
+
|
|
110
|
+
def parse(args)
|
|
111
|
+
options = { format: "text" }
|
|
112
|
+
parser = OptionParser.new do |opts|
|
|
113
|
+
opts.banner = BANNER
|
|
114
|
+
opts.separator("Options:")
|
|
115
|
+
opts.on("-c", "--config FILE", "Configuration file (default: #{Config::FILE})") { |value| options[:config] = value }
|
|
116
|
+
opts.on("-f", "--format FORMAT", REPORTERS.keys, "Report format: text, json or sarif") { |value| options[:format] = value }
|
|
117
|
+
opts.on("-o", "--output FILE", "Write the report to FILE") { |value| options[:output] = value }
|
|
118
|
+
opts.on("--all", "audit: include baselined findings") { options[:all] = true }
|
|
119
|
+
opts.on("--dry-run", "scrub, trim: show what would change, write nothing") { options[:dry_run] = true }
|
|
120
|
+
opts.on("--only-responses", "scrub: leave recorded requests untouched") { options[:only_responses] = true }
|
|
121
|
+
opts.on("--min-size KB", Integer, "trim: smallest body or field to trim (default 100)") { |value| options[:min_size] = value }
|
|
122
|
+
opts.on("--usage-dir DIR", "Where CassetteGuard::Usage.track! wrote its files") { |value| options[:usage_dir] = value }
|
|
123
|
+
opts.on("--baseline FILE", "Baseline file (default .cassette_guard_baseline.yml)") { |value| options[:baseline] = value }
|
|
124
|
+
opts.on("-h", "--help", "Show this help") { options[:help] = true }
|
|
125
|
+
end
|
|
126
|
+
@help = parser.help
|
|
127
|
+
parser.parse!(args)
|
|
128
|
+
options[:paths] = args
|
|
129
|
+
options
|
|
130
|
+
end
|
|
131
|
+
|
|
132
|
+
def load_config(options)
|
|
133
|
+
overrides = {
|
|
134
|
+
"cassettes" => options[:paths].empty? ? nil : options[:paths],
|
|
135
|
+
"usage_dir" => options[:usage_dir],
|
|
136
|
+
"baseline" => options[:baseline],
|
|
137
|
+
"trim_min_size_kb" => options[:min_size],
|
|
138
|
+
}
|
|
139
|
+
Config.load(root: @root, path: options[:config], overrides:)
|
|
140
|
+
end
|
|
141
|
+
|
|
142
|
+
def render(report, shown, options)
|
|
143
|
+
groups = FindingGroup.from(shown)
|
|
144
|
+
hidden = FindingGroup.from(report.findings).size - groups.size
|
|
145
|
+
REPORTERS.fetch(options[:format]).new(@config).render(report, groups, hidden:)
|
|
146
|
+
end
|
|
147
|
+
|
|
148
|
+
def emit(text, options)
|
|
149
|
+
return @out.print(text) unless options[:output]
|
|
150
|
+
|
|
151
|
+
File.write(options[:output], text)
|
|
152
|
+
end
|
|
153
|
+
|
|
154
|
+
def help
|
|
155
|
+
@out.puts(@help)
|
|
156
|
+
0
|
|
157
|
+
end
|
|
158
|
+
|
|
159
|
+
def version
|
|
160
|
+
@out.puts("cassette_guard #{VERSION}")
|
|
161
|
+
0
|
|
162
|
+
end
|
|
163
|
+
|
|
164
|
+
class ScrubCommand
|
|
165
|
+
def initialize(config, out, options)
|
|
166
|
+
@config = config
|
|
167
|
+
@out = out
|
|
168
|
+
@dry_run = options[:dry_run]
|
|
169
|
+
@scanner = Scanner.new(config)
|
|
170
|
+
@scrubber = Scrubber.new(config, only_responses: options[:only_responses])
|
|
171
|
+
@baseline = Baseline.load(config.baseline_path)
|
|
172
|
+
end
|
|
173
|
+
|
|
174
|
+
def call(files)
|
|
175
|
+
report = @scanner.scan(files)
|
|
176
|
+
targets = report.findings.select { |finding| finding.sensitive? && !@baseline.include?(finding) }
|
|
177
|
+
if targets.empty?
|
|
178
|
+
@out.puts("Nothing to scrub in #{Format.plural(report.cassettes.size, "cassette")}.")
|
|
179
|
+
return 0
|
|
180
|
+
end
|
|
181
|
+
|
|
182
|
+
plans = targets.group_by(&:cassette).sort.to_h { |cassette, findings| [cassette, scrub(cassette, findings)] }
|
|
183
|
+
@baseline.save! unless @dry_run
|
|
184
|
+
summarize(plans)
|
|
185
|
+
end
|
|
186
|
+
|
|
187
|
+
private
|
|
188
|
+
|
|
189
|
+
def scrub(relative_path, findings)
|
|
190
|
+
cassette = Cassette.load(File.expand_path(relative_path, @config.root))
|
|
191
|
+
recorder = cassette.recorder
|
|
192
|
+
@out.puts(relative_path)
|
|
193
|
+
return skip(findings, recorder, cassette.unwritable_reason) unless cassette.writable?
|
|
194
|
+
|
|
195
|
+
plan = @scrubber.scrub(cassette, findings)
|
|
196
|
+
print_plan(plan)
|
|
197
|
+
begin
|
|
198
|
+
if @dry_run
|
|
199
|
+
cassette.dump
|
|
200
|
+
return { plan:, leftovers: [], recorder: }
|
|
201
|
+
end
|
|
202
|
+
cassette.save! if cassette.changed?
|
|
203
|
+
rescue Error => e
|
|
204
|
+
return skip(findings, recorder, e.message)
|
|
205
|
+
end
|
|
206
|
+
fakes = plan.to_set(&:fake)
|
|
207
|
+
leftovers = []
|
|
208
|
+
@scanner.scan_cassette(cassette).select(&:sensitive?).reject { |finding| @baseline.include?(finding) }.each do |finding|
|
|
209
|
+
if fakes.include?(finding.value)
|
|
210
|
+
@baseline.add(finding, reason: "scrubbed")
|
|
211
|
+
else
|
|
212
|
+
leftovers << finding
|
|
213
|
+
end
|
|
214
|
+
end
|
|
215
|
+
{ plan:, leftovers:, recorder: }
|
|
216
|
+
end
|
|
217
|
+
|
|
218
|
+
def skip(findings, recorder, reason)
|
|
219
|
+
@out.puts(" not written: #{reason}")
|
|
220
|
+
{ plan: [], leftovers: findings, recorder: }
|
|
221
|
+
end
|
|
222
|
+
|
|
223
|
+
def print_plan(plan)
|
|
224
|
+
width = plan.map { |replacement| replacement.finding.where.length }.max
|
|
225
|
+
plan.sort_by { |replacement| [replacement.finding.interaction || -1, replacement.finding.location.to_s] }.each do |replacement|
|
|
226
|
+
finding = replacement.finding
|
|
227
|
+
fake = replacement.fake.length > 40 ? "#{replacement.fake[0, 32]}…" : replacement.fake
|
|
228
|
+
@out.puts(format(" %-#{width}s %s → %s", finding.where, finding.detail, fake))
|
|
229
|
+
end
|
|
230
|
+
end
|
|
231
|
+
|
|
232
|
+
def summarize(plans)
|
|
233
|
+
values = plans.values.sum { |result| result[:plan].size }
|
|
234
|
+
leftovers = plans.values.flat_map { |result| result[:leftovers] }
|
|
235
|
+
verb = @dry_run ? "Would scrub" : "Scrubbed"
|
|
236
|
+
@out.puts("\n#{verb} #{Format.plural(values, "value")} in #{Format.plural(plans.size, "cassette")}.")
|
|
237
|
+
unless @dry_run
|
|
238
|
+
@out.puts("Recorded the fakes in #{@config.relative(@baseline.path)} so later runs leave them alone; commit it.")
|
|
239
|
+
@out.puts("Run your tests now. If one fails because no recorded request matches (VCR's " \
|
|
240
|
+
"UnhandledHTTPRequestError, vcrpy's CannotOverwriteExistingCassetteException), the value comes " \
|
|
241
|
+
"from the test itself and requests are matched on their body: change the value in the test, or " \
|
|
242
|
+
"stop matching on the body.")
|
|
243
|
+
end
|
|
244
|
+
salt_warning(plans)
|
|
245
|
+
filter_hint(plans, :vcr)
|
|
246
|
+
filter_hint(plans, :vcrpy)
|
|
247
|
+
report_leftovers(leftovers)
|
|
248
|
+
end
|
|
249
|
+
|
|
250
|
+
def salt_warning(plans)
|
|
251
|
+
return unless @config.default_salt?
|
|
252
|
+
return unless plans.values.any? { |result| result[:plan].any? { |replacement| replacement.finding.rule == :pii } }
|
|
253
|
+
|
|
254
|
+
@out.puts("Warning: fakes are derived from the originals with the built-in salt, so short values such as " \
|
|
255
|
+
"birth dates can be recovered by brute force. Set CASSETTE_GUARD_SALT to a private value.")
|
|
256
|
+
end
|
|
257
|
+
|
|
258
|
+
def filter_hint(plans, recorder)
|
|
259
|
+
replacements = plans.values.select { |result| result[:recorder] == recorder }.flat_map { |result| result[:plan] }
|
|
260
|
+
headers = replacements.select(&:header?).uniq { |replacement| replacement.header_name.to_s.downcase }
|
|
261
|
+
return if headers.empty?
|
|
262
|
+
|
|
263
|
+
@out.puts("\nKeep these headers out of the next recording too:\n\n")
|
|
264
|
+
recorder == :vcrpy ? vcrpy_filters(headers) : vcr_filters(headers)
|
|
265
|
+
end
|
|
266
|
+
|
|
267
|
+
def vcr_filters(headers)
|
|
268
|
+
@out.puts(" VCR.configure do |c|")
|
|
269
|
+
headers.each do |replacement|
|
|
270
|
+
name = replacement.header_name
|
|
271
|
+
side = replacement.finding.location.start_with?("response") ? "response" : "request"
|
|
272
|
+
placeholder = "<#{name.upcase.tr("-", "_")}>"
|
|
273
|
+
@out.puts(" c.filter_sensitive_data(#{placeholder.inspect}) { |i| i.#{side}.headers[#{name.inspect}]&.first }")
|
|
274
|
+
end
|
|
275
|
+
@out.puts(" end")
|
|
276
|
+
end
|
|
277
|
+
|
|
278
|
+
def vcrpy_filters(headers)
|
|
279
|
+
requests, responses = headers.partition { |replacement| replacement.finding.location.start_with?("request") }
|
|
280
|
+
request_names = requests.map { |replacement| [replacement.header_name, "<#{replacement.header_name.upcase.tr("-", "_")}>"] }
|
|
281
|
+
response_names = responses.map(&:header_name)
|
|
282
|
+
@out.puts(" # conftest.py (pytest-recording or pytest-vcr)")
|
|
283
|
+
unless response_names.empty?
|
|
284
|
+
@out.puts(" def _drop_response_headers(response):")
|
|
285
|
+
@out.puts(" for name in #{python_tuple(response_names)}:")
|
|
286
|
+
@out.puts(" response[\"headers\"].pop(name, None)")
|
|
287
|
+
@out.puts(" return response\n\n")
|
|
288
|
+
end
|
|
289
|
+
@out.puts(" @pytest.fixture(scope=\"module\")\n def vcr_config():\n return {")
|
|
290
|
+
@out.puts(" \"filter_headers\": [#{request_names.map { |pair| python_tuple(pair) }.join(", ")}],") unless request_names.empty?
|
|
291
|
+
@out.puts(" \"before_record_response\": _drop_response_headers,") unless response_names.empty?
|
|
292
|
+
@out.puts(" }")
|
|
293
|
+
end
|
|
294
|
+
|
|
295
|
+
def python_tuple(values)
|
|
296
|
+
items = values.map { |value| value.inspect }
|
|
297
|
+
"(#{items.join(", ")}#{"," if items.one?})"
|
|
298
|
+
end
|
|
299
|
+
|
|
300
|
+
def report_leftovers(leftovers)
|
|
301
|
+
return 0 if leftovers.empty?
|
|
302
|
+
|
|
303
|
+
@out.puts("\nCould not scrub #{Format.plural(leftovers.size, "value")}; left as is:")
|
|
304
|
+
leftovers.each { |finding| @out.puts(" #{finding.cassette} #{finding.where} #{finding.detail}") }
|
|
305
|
+
1
|
|
306
|
+
end
|
|
307
|
+
end
|
|
308
|
+
end
|
|
309
|
+
end
|
|
@@ -0,0 +1,106 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module CassetteGuard
|
|
4
|
+
class Config
|
|
5
|
+
FILE = ".cassette_guard.yml"
|
|
6
|
+
DEFAULT_SALT = "cassette_guard-default-salt"
|
|
7
|
+
DEFAULT_CASSETTE_DIRS = %w[
|
|
8
|
+
spec/fixtures/vcr_cassettes spec/vcr_cassettes spec/cassettes spec/fixtures/cassettes
|
|
9
|
+
test/vcr_cassettes test/cassettes test/fixtures/vcr_cassettes
|
|
10
|
+
tests/cassettes tests/fixtures/cassettes tests/fixtures/vcr_cassettes tests/vcr_cassettes
|
|
11
|
+
].freeze
|
|
12
|
+
|
|
13
|
+
attr_reader :root, :paths, :max_file_size, :max_body_size, :max_age_days, :trim_min_size,
|
|
14
|
+
:usage_dir, :baseline_path, :allowed_values, :allowed_email_patterns, :allowed_keys,
|
|
15
|
+
:extra_pii_keys, :extra_secret_keys, :extra_secret_headers, :extra_secret_cookies
|
|
16
|
+
|
|
17
|
+
def self.load(root: Dir.pwd, path: nil, overrides: {})
|
|
18
|
+
file = File.expand_path(path || FILE, root)
|
|
19
|
+
raise Error, "#{path}: no such file" if path && !File.exist?(file)
|
|
20
|
+
|
|
21
|
+
data = File.exist?(file) ? YAML.safe_load_file(file) || {} : {}
|
|
22
|
+
raise Error, "#{file}: expected a mapping" unless data.is_a?(Hash)
|
|
23
|
+
|
|
24
|
+
new(root:, data: data.merge(overrides.compact))
|
|
25
|
+
end
|
|
26
|
+
|
|
27
|
+
def initialize(root:, data: {})
|
|
28
|
+
@root = utf8(File.expand_path(root))
|
|
29
|
+
@paths = Array(data["cassettes"] || default_paths)
|
|
30
|
+
@max_file_size = Integer(data.fetch("max_file_size_kb", 1024)) * 1024
|
|
31
|
+
@max_body_size = Integer(data.fetch("max_body_size_kb", 256)) * 1024
|
|
32
|
+
@max_age_days = Integer(data.fetch("max_age_days", 365))
|
|
33
|
+
@trim_min_size = Integer(data.fetch("trim_min_size_kb", 100)) * 1024
|
|
34
|
+
raise Error, "trim_min_size_kb (--min-size) must be at least 1" if @trim_min_size < 1024
|
|
35
|
+
@usage_dir = data.fetch("usage_dir", "tmp/cassette_guard")
|
|
36
|
+
@baseline_path = File.expand_path(data.fetch("baseline", ".cassette_guard_baseline.yml"), @root)
|
|
37
|
+
@salt = data["salt"]
|
|
38
|
+
load_allowlist(data.fetch("allow", nil) || {})
|
|
39
|
+
@extra_pii_keys = normalized_list(data["pii_keys"])
|
|
40
|
+
@extra_secret_keys = normalized_list(data["secret_keys"])
|
|
41
|
+
@extra_secret_headers = Array(data["secret_headers"]).map { |name| name.to_s.downcase }
|
|
42
|
+
@extra_secret_cookies = Array(data["secret_cookies"]).map { |name| name.to_s.downcase }
|
|
43
|
+
end
|
|
44
|
+
|
|
45
|
+
def salt
|
|
46
|
+
ENV.fetch("CASSETTE_GUARD_SALT", nil) || @salt || DEFAULT_SALT
|
|
47
|
+
end
|
|
48
|
+
|
|
49
|
+
def default_salt?
|
|
50
|
+
salt == DEFAULT_SALT
|
|
51
|
+
end
|
|
52
|
+
|
|
53
|
+
# Without a UTF-8 locale, Dir.glob returns raw bytes for a name like "Schöne.yml", which then breaks
|
|
54
|
+
# every report. File names are UTF-8 on every platform cassettes come from, so say so.
|
|
55
|
+
def cassette_files
|
|
56
|
+
paths.flat_map { |path| expand(path) }.map { |path| utf8(path) }.uniq.sort
|
|
57
|
+
end
|
|
58
|
+
|
|
59
|
+
def relative(path)
|
|
60
|
+
full = File.expand_path(path, root)
|
|
61
|
+
full.start_with?("#{root}/") ? full.delete_prefix("#{root}/") : full
|
|
62
|
+
end
|
|
63
|
+
|
|
64
|
+
def allowed_value?(value)
|
|
65
|
+
allowed_values.include?(value.to_s)
|
|
66
|
+
end
|
|
67
|
+
|
|
68
|
+
def allowed_key?(key)
|
|
69
|
+
allowed_keys.include?(Rules.normalize_key(key))
|
|
70
|
+
end
|
|
71
|
+
|
|
72
|
+
def allowed_email?(email)
|
|
73
|
+
allowed_email_patterns.any? { |pattern| File.fnmatch?(pattern, email, File::FNM_CASEFOLD) }
|
|
74
|
+
end
|
|
75
|
+
|
|
76
|
+
private
|
|
77
|
+
|
|
78
|
+
def utf8(path)
|
|
79
|
+
path.dup.force_encoding(Encoding::UTF_8)
|
|
80
|
+
end
|
|
81
|
+
|
|
82
|
+
def load_allowlist(allow)
|
|
83
|
+
raise Error, "#{FILE}: `allow` must be a mapping" unless allow.is_a?(Hash)
|
|
84
|
+
|
|
85
|
+
@allowed_values = Array(allow["values"]).map(&:to_s)
|
|
86
|
+
@allowed_email_patterns = Rules::ALLOWED_EMAIL_PATTERNS + Array(allow["emails"]).map(&:to_s)
|
|
87
|
+
@allowed_keys = normalized_list(allow["keys"])
|
|
88
|
+
end
|
|
89
|
+
|
|
90
|
+
def normalized_list(values)
|
|
91
|
+
Array(values).map { |value| Rules.normalize_key(value) }
|
|
92
|
+
end
|
|
93
|
+
|
|
94
|
+
def default_paths
|
|
95
|
+
DEFAULT_CASSETTE_DIRS.select { |dir| File.directory?(File.join(@root, dir)) }
|
|
96
|
+
end
|
|
97
|
+
|
|
98
|
+
def expand(path)
|
|
99
|
+
full = File.expand_path(path, root)
|
|
100
|
+
return Dir.glob(File.join(full, "**", "*.{yml,yaml,json}")) if File.directory?(full)
|
|
101
|
+
return [full] if File.file?(full)
|
|
102
|
+
|
|
103
|
+
Dir.glob(full).select { |match| File.file?(match) }
|
|
104
|
+
end
|
|
105
|
+
end
|
|
106
|
+
end
|
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module CassetteGuard
|
|
4
|
+
module Detectors
|
|
5
|
+
class Age
|
|
6
|
+
def initialize(config, now: Time.now)
|
|
7
|
+
@config = config
|
|
8
|
+
@now = now
|
|
9
|
+
end
|
|
10
|
+
|
|
11
|
+
def call(cassette, relative_path)
|
|
12
|
+
oldest = cassette.interactions.filter_map { |interaction| parse(recorded_at(interaction)) }.min
|
|
13
|
+
return [] unless oldest
|
|
14
|
+
|
|
15
|
+
days = ((@now - oldest) / 86_400).floor
|
|
16
|
+
return [] if days <= @config.max_age_days
|
|
17
|
+
|
|
18
|
+
message = "recorded #{oldest.utc.strftime("%Y-%m-%d")} (#{Format.number(days)} days ago)"
|
|
19
|
+
[Finding.new(rule: :age, kind: "recorded_at", cassette: relative_path, message:)]
|
|
20
|
+
end
|
|
21
|
+
|
|
22
|
+
private
|
|
23
|
+
|
|
24
|
+
# vcrpy has no recorded_at; the response's Date header says when it was recorded.
|
|
25
|
+
def recorded_at(interaction)
|
|
26
|
+
return interaction["recorded_at"] if interaction["recorded_at"]
|
|
27
|
+
|
|
28
|
+
response = interaction["response"]
|
|
29
|
+
Headers.get(response["headers"], "date") if response.is_a?(Hash)
|
|
30
|
+
end
|
|
31
|
+
|
|
32
|
+
def parse(value)
|
|
33
|
+
case value
|
|
34
|
+
when Time then value
|
|
35
|
+
when Date then value.to_time
|
|
36
|
+
when String
|
|
37
|
+
begin
|
|
38
|
+
Time.httpdate(value)
|
|
39
|
+
rescue ArgumentError
|
|
40
|
+
Time.parse(value) rescue nil # rubocop:disable Style/RescueModifier
|
|
41
|
+
end
|
|
42
|
+
end
|
|
43
|
+
end
|
|
44
|
+
end
|
|
45
|
+
end
|
|
46
|
+
end
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module CassetteGuard
|
|
4
|
+
module Detectors
|
|
5
|
+
# A key name like `birth_date` is enough on its own; without one, only values that validate
|
|
6
|
+
# (an email, an IBAN with a correct checksum) count.
|
|
7
|
+
class Pii
|
|
8
|
+
def initialize(config)
|
|
9
|
+
@config = config
|
|
10
|
+
end
|
|
11
|
+
|
|
12
|
+
def field(field)
|
|
13
|
+
value = field.value.to_s.strip
|
|
14
|
+
return [] if Rules.placeholder?(value) || @config.allowed_value?(value)
|
|
15
|
+
return keyed(field.key, value) if field.key && pii_key?(field.key) && !@config.allowed_key?(field.key)
|
|
16
|
+
|
|
17
|
+
patterns(value)
|
|
18
|
+
end
|
|
19
|
+
|
|
20
|
+
private
|
|
21
|
+
|
|
22
|
+
def keyed(key, value)
|
|
23
|
+
return [] if value.length < 2 || %w[true false].include?(value)
|
|
24
|
+
return [] if Rules.email?(value) && @config.allowed_email?(value)
|
|
25
|
+
return [] if Rules.card_key?(key) && !real_card?(value)
|
|
26
|
+
|
|
27
|
+
[["key:#{key}", value]]
|
|
28
|
+
end
|
|
29
|
+
|
|
30
|
+
def patterns(text)
|
|
31
|
+
emails = text.scan(Rules::EMAIL).reject { |email| @config.allowed_email?(email) }.map { |email| ["email", email] }
|
|
32
|
+
ibans = text.scan(Rules::IBAN).select { |iban| Rules.iban?(iban) }.map { |iban| ["iban", iban] }
|
|
33
|
+
(emails + ibans).uniq
|
|
34
|
+
end
|
|
35
|
+
|
|
36
|
+
def real_card?(value)
|
|
37
|
+
Rules.luhn?(value) && !Rules::TEST_CARD_NUMBERS.include?(value.delete(" -"))
|
|
38
|
+
end
|
|
39
|
+
|
|
40
|
+
def pii_key?(key)
|
|
41
|
+
Rules.pii_key?(key) || @config.extra_pii_keys.include?(Rules.normalize_key(key))
|
|
42
|
+
end
|
|
43
|
+
end
|
|
44
|
+
end
|
|
45
|
+
end
|
|
@@ -0,0 +1,83 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module CassetteGuard
|
|
4
|
+
module Detectors
|
|
5
|
+
# Returns [[kind, value], ...]; the value is the secret part only, so it can be replaced everywhere it appears.
|
|
6
|
+
class Secret
|
|
7
|
+
AUTH_SCHEME = /\A(?:Bearer|Basic|Token|Digest|ApiKey|Bot|OAuth)\s+/i
|
|
8
|
+
# Only cookies whose name says they authenticate someone. Analytics ids, load-balancer stickiness,
|
|
9
|
+
# correlation ids and preferences are long but harmless, and CSRF tokens die with the session.
|
|
10
|
+
SESSION_COOKIE = /sess|sid|token|auth|jwt|login|remember|secret|refresh|access|identity|saml|oauth/i
|
|
11
|
+
NOT_SESSION_COOKIE = /csrf|xsrf/i
|
|
12
|
+
|
|
13
|
+
def initialize(config)
|
|
14
|
+
@config = config
|
|
15
|
+
end
|
|
16
|
+
|
|
17
|
+
def field(field)
|
|
18
|
+
key = field.key
|
|
19
|
+
if key && secret_key?(key) && !@config.allowed_key?(key)
|
|
20
|
+
value = field.value.to_s.strip
|
|
21
|
+
return secret?(value) ? [["key:#{key}", value]] : []
|
|
22
|
+
end
|
|
23
|
+
tokens(field.value.to_s)
|
|
24
|
+
end
|
|
25
|
+
|
|
26
|
+
def header(name, value)
|
|
27
|
+
name = name.to_s.downcase
|
|
28
|
+
return tokens(value) unless secret_header?(name)
|
|
29
|
+
|
|
30
|
+
candidates =
|
|
31
|
+
case name
|
|
32
|
+
when "cookie" then cookies(value)
|
|
33
|
+
when "set-cookie" then cookies(value.split(";").first.to_s)
|
|
34
|
+
when "authorization", "proxy-authorization" then [["header:#{name}", value.sub(AUTH_SCHEME, "").strip]]
|
|
35
|
+
else [["header:#{name}", value.strip]]
|
|
36
|
+
end
|
|
37
|
+
candidates.select { |_kind, candidate| secret?(candidate) }
|
|
38
|
+
end
|
|
39
|
+
|
|
40
|
+
def uri_password(password)
|
|
41
|
+
return [] if password.nil? || password.empty? || Rules.placeholder?(password) || @config.allowed_value?(password)
|
|
42
|
+
|
|
43
|
+
[["uri:password", password]]
|
|
44
|
+
end
|
|
45
|
+
|
|
46
|
+
private
|
|
47
|
+
|
|
48
|
+
def tokens(text)
|
|
49
|
+
Rules::TOKEN_PATTERNS.flat_map { |kind, pattern| text.scan(pattern).map { |match| [kind, match] } }
|
|
50
|
+
.reject { |_kind, match| Rules.placeholder?(match) || @config.allowed_value?(match) }
|
|
51
|
+
end
|
|
52
|
+
|
|
53
|
+
def cookies(text)
|
|
54
|
+
text.split(/;\s*/).flat_map do |pair|
|
|
55
|
+
name, value = pair.split("=", 2)
|
|
56
|
+
next [] unless value
|
|
57
|
+
|
|
58
|
+
name = name.strip
|
|
59
|
+
value = value.strip
|
|
60
|
+
session_cookie?(name) && value.length >= 8 ? [["cookie:#{name}", value]] : tokens(value)
|
|
61
|
+
end
|
|
62
|
+
end
|
|
63
|
+
|
|
64
|
+
def session_cookie?(name)
|
|
65
|
+
return true if @config.extra_secret_cookies.include?(name.downcase)
|
|
66
|
+
|
|
67
|
+
name.match?(SESSION_COOKIE) && !name.match?(NOT_SESSION_COOKIE)
|
|
68
|
+
end
|
|
69
|
+
|
|
70
|
+
def secret?(value)
|
|
71
|
+
value.length >= 4 && !Rules.placeholder?(value) && !@config.allowed_value?(value)
|
|
72
|
+
end
|
|
73
|
+
|
|
74
|
+
def secret_key?(key)
|
|
75
|
+
Rules.secret_key?(key) || @config.extra_secret_keys.include?(Rules.normalize_key(key))
|
|
76
|
+
end
|
|
77
|
+
|
|
78
|
+
def secret_header?(name)
|
|
79
|
+
Rules::SECRET_HEADERS.include?(name) || @config.extra_secret_headers.include?(name)
|
|
80
|
+
end
|
|
81
|
+
end
|
|
82
|
+
end
|
|
83
|
+
end
|