cassette_guard 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/CHANGELOG.md +15 -0
- data/LICENSE.txt +21 -0
- data/README.md +454 -0
- data/exe/cassette_guard +6 -0
- data/lib/cassette_guard/baseline.rb +59 -0
- data/lib/cassette_guard/body.rb +100 -0
- data/lib/cassette_guard/cassette.rb +81 -0
- data/lib/cassette_guard/cli.rb +309 -0
- data/lib/cassette_guard/config.rb +106 -0
- data/lib/cassette_guard/detectors/age.rb +46 -0
- data/lib/cassette_guard/detectors/pii.rb +45 -0
- data/lib/cassette_guard/detectors/secret.rb +83 -0
- data/lib/cassette_guard/detectors/size.rb +28 -0
- data/lib/cassette_guard/detectors/usage.rb +45 -0
- data/lib/cassette_guard/encodings.rb +79 -0
- data/lib/cassette_guard/fake_value.rb +103 -0
- data/lib/cassette_guard/fields.rb +74 -0
- data/lib/cassette_guard/finding.rb +33 -0
- data/lib/cassette_guard/finding_group.rb +37 -0
- data/lib/cassette_guard/format.rb +37 -0
- data/lib/cassette_guard/headers.rb +33 -0
- data/lib/cassette_guard/line_locator.rb +37 -0
- data/lib/cassette_guard/py_yaml.rb +228 -0
- data/lib/cassette_guard/reporters/json.rb +24 -0
- data/lib/cassette_guard/reporters/sarif.rb +67 -0
- data/lib/cassette_guard/reporters/text.rb +76 -0
- data/lib/cassette_guard/rules.rb +127 -0
- data/lib/cassette_guard/scanner.rb +131 -0
- data/lib/cassette_guard/scrubber.rb +138 -0
- data/lib/cassette_guard/trimmer.rb +70 -0
- data/lib/cassette_guard/usage.rb +47 -0
- data/lib/cassette_guard/version.rb +5 -0
- data/lib/cassette_guard.rb +40 -0
- metadata +84 -0
|
@@ -0,0 +1,228 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "securerandom"
|
|
4
|
+
|
|
5
|
+
module CassetteGuard
|
|
6
|
+
# Reads and writes YAML exactly as vcrpy does (PyYAML's resolver and representer on libyaml's emitter,
|
|
7
|
+
# allow_unicode off), so a rewritten cassette differs from the original only where a value changed.
|
|
8
|
+
#
|
|
9
|
+
# Reading: Psych's own resolver follows Ruby's rules ("1,000" is an Integer, "yEs" is true, ":x" a Symbol),
|
|
10
|
+
# so plain scalars are resolved here with PyYAML's rules instead. Anything PyYAML would not load as a
|
|
11
|
+
# string is kept as a Plain holding its exact source text, and written back unchanged.
|
|
12
|
+
#
|
|
13
|
+
# Writing: Psych drives the same libyaml emitter, so structure, quoting and line folding come out
|
|
14
|
+
# identical once the nodes carry PyYAML's tags and implicit flags. The one thing Psych cannot switch
|
|
15
|
+
# off is unicode output: PyYAML double-quotes and escapes every scalar with a non-ASCII character.
|
|
16
|
+
# Those go through Psych as unique placeholders and are rendered by a port of libyaml's double-quoted
|
|
17
|
+
# writer.
|
|
18
|
+
module PyYaml
|
|
19
|
+
BEST_WIDTH = 80
|
|
20
|
+
INDENT = 2
|
|
21
|
+
TAG = "tag:yaml.org,2002:"
|
|
22
|
+
STR = "#{TAG}str".freeze
|
|
23
|
+
|
|
24
|
+
# PyYAML's implicit resolvers (YAML 1.1), in PyYAML's order: the first match decides the type.
|
|
25
|
+
RESOLVERS = {
|
|
26
|
+
"bool" => /\A(?:yes|Yes|YES|no|No|NO|true|True|TRUE|false|False|FALSE|on|On|ON|off|Off|OFF)\z/,
|
|
27
|
+
"float" => /\A(?:[-+]?[0-9][0-9_]*\.[0-9_]*(?:[eE][-+][0-9]+)?|\.[0-9][0-9_]*(?:[eE][-+][0-9]+)?|
|
|
28
|
+
[-+]?[0-9][0-9_]*(?::[0-5]?[0-9])+\.[0-9_]*|[-+]?\.(?:inf|Inf|INF)|\.(?:nan|NaN|NAN))\z/x,
|
|
29
|
+
"int" => /\A(?:[-+]?0b[0-1_]+|[-+]?0[0-7_]+|[-+]?(?:0|[1-9][0-9_]*)|[-+]?0x[0-9a-fA-F_]+|
|
|
30
|
+
[-+]?[1-9][0-9_]*(?::[0-5]?[0-9])+)\z/x,
|
|
31
|
+
"merge" => /\A<<\z/,
|
|
32
|
+
"null" => /\A(?:~|null|Null|NULL|)\z/,
|
|
33
|
+
"timestamp" => /\A(?:[0-9]{4}-[0-9]{2}-[0-9]{2}|
|
|
34
|
+
[0-9]{4}-[0-9][0-9]?-[0-9][0-9]?(?:[Tt]|[\ \t]+)[0-9][0-9]?:[0-9]{2}:[0-9]{2}(?:\.[0-9]*)?
|
|
35
|
+
(?:[\ \t]*(?:Z|[-+][0-9][0-9]?(?::[0-9]{2})?))?)\z/x,
|
|
36
|
+
"value" => /\A=\z/,
|
|
37
|
+
}.freeze
|
|
38
|
+
|
|
39
|
+
ESCAPES = {
|
|
40
|
+
0x00 => "0", 0x07 => "a", 0x08 => "b", 0x09 => "t", 0x0A => "n", 0x0B => "v", 0x0C => "f", 0x0D => "r",
|
|
41
|
+
0x1B => "e", 0x22 => '"', 0x5C => "\\", 0x85 => "N", 0xA0 => "_", 0x2028 => "L", 0x2029 => "P",
|
|
42
|
+
}.freeze
|
|
43
|
+
|
|
44
|
+
# A scalar PyYAML would load as something other than a string, kept as written.
|
|
45
|
+
Plain = Data.define(:text, :tag) do
|
|
46
|
+
def to_s = text
|
|
47
|
+
end
|
|
48
|
+
|
|
49
|
+
module_function
|
|
50
|
+
|
|
51
|
+
def resolve(text)
|
|
52
|
+
RESOLVERS.find { |_type, pattern| pattern.match?(text) }&.first
|
|
53
|
+
end
|
|
54
|
+
|
|
55
|
+
def vcrpy?(root)
|
|
56
|
+
root.is_a?(Psych::Nodes::Mapping) &&
|
|
57
|
+
root.children.each_slice(2).any? { |key, value| key.is_a?(Psych::Nodes::Scalar) && key.value == "interactions" && value.is_a?(Psych::Nodes::Sequence) }
|
|
58
|
+
end
|
|
59
|
+
|
|
60
|
+
# Returns [data, tags_it_cannot_write].
|
|
61
|
+
def load(root)
|
|
62
|
+
loader = Loader.new
|
|
63
|
+
[loader.build(root), loader.unsupported.uniq]
|
|
64
|
+
end
|
|
65
|
+
|
|
66
|
+
class Loader
|
|
67
|
+
COLLECTION_TAGS = { Psych::Nodes::Mapping => "#{TAG}map", Psych::Nodes::Sequence => "#{TAG}seq" }.freeze
|
|
68
|
+
|
|
69
|
+
attr_reader :unsupported
|
|
70
|
+
|
|
71
|
+
def initialize
|
|
72
|
+
@anchors = {}
|
|
73
|
+
@unsupported = []
|
|
74
|
+
end
|
|
75
|
+
|
|
76
|
+
def build(node)
|
|
77
|
+
return @anchors.fetch(node.anchor) if node.is_a?(Psych::Nodes::Alias)
|
|
78
|
+
|
|
79
|
+
value =
|
|
80
|
+
case node
|
|
81
|
+
when Psych::Nodes::Mapping then node.children.each_slice(2).to_h { |key, child| [build(key), build(child)] }
|
|
82
|
+
when Psych::Nodes::Sequence then node.children.map { |child| build(child) }
|
|
83
|
+
else scalar(node)
|
|
84
|
+
end
|
|
85
|
+
check_collection_tag(node)
|
|
86
|
+
@anchors[node.anchor] = value if node.anchor
|
|
87
|
+
value
|
|
88
|
+
end
|
|
89
|
+
|
|
90
|
+
private
|
|
91
|
+
|
|
92
|
+
def check_collection_tag(node)
|
|
93
|
+
expected = COLLECTION_TAGS[node.class]
|
|
94
|
+
@unsupported << node.tag if expected && node.tag && node.tag != expected
|
|
95
|
+
end
|
|
96
|
+
|
|
97
|
+
def scalar(node)
|
|
98
|
+
case node.tag
|
|
99
|
+
when "#{TAG}binary" then return node.value.unpack1("m").b
|
|
100
|
+
when nil, STR, "!" then nil
|
|
101
|
+
else @unsupported << node.tag
|
|
102
|
+
end
|
|
103
|
+
return node.value if node.tag || node.style != Psych::Nodes::Scalar::PLAIN
|
|
104
|
+
|
|
105
|
+
type = PyYaml.resolve(node.value)
|
|
106
|
+
return node.value if type.nil?
|
|
107
|
+
return nil if type == "null" && node.value == "null"
|
|
108
|
+
|
|
109
|
+
Plain.new(text: node.value, tag: "#{TAG}#{type}")
|
|
110
|
+
end
|
|
111
|
+
end
|
|
112
|
+
|
|
113
|
+
def dump(data)
|
|
114
|
+
placeholders = {}
|
|
115
|
+
prefix = "cgq#{SecureRandom.hex(6)}x"
|
|
116
|
+
document = Psych::Nodes::Document.new([], [], true)
|
|
117
|
+
document.children << node(data, placeholders, prefix)
|
|
118
|
+
stream = Psych::Nodes::Stream.new
|
|
119
|
+
stream.children << document
|
|
120
|
+
yaml = stream.to_yaml(nil, line_width: BEST_WIDTH, indentation: INDENT)
|
|
121
|
+
placeholders.reduce(yaml) { |text, (token, (value, key))| splice(text, token, value, key) }
|
|
122
|
+
end
|
|
123
|
+
|
|
124
|
+
def node(value, placeholders, prefix, key: false)
|
|
125
|
+
case value
|
|
126
|
+
when Hash
|
|
127
|
+
mapping = Psych::Nodes::Mapping.new(nil, nil, true, Psych::Nodes::Mapping::BLOCK)
|
|
128
|
+
value.each do |child_key, child|
|
|
129
|
+
mapping.children << node(child_key, placeholders, prefix, key: true) << node(child, placeholders, prefix)
|
|
130
|
+
end
|
|
131
|
+
mapping
|
|
132
|
+
when Array
|
|
133
|
+
sequence = Psych::Nodes::Sequence.new(nil, nil, true, Psych::Nodes::Sequence::BLOCK)
|
|
134
|
+
value.each { |child| sequence.children << node(child, placeholders, prefix) }
|
|
135
|
+
sequence
|
|
136
|
+
when Plain then scalar(value.text, value.tag, true, false, Psych::Nodes::Scalar::PLAIN)
|
|
137
|
+
when nil then scalar("null", "#{TAG}null", true, false)
|
|
138
|
+
when true, false then scalar(value.to_s, "#{TAG}bool", true, false)
|
|
139
|
+
when Integer then scalar(value.to_s, "#{TAG}int", true, false)
|
|
140
|
+
when Float then scalar(float_text(value), "#{TAG}float", true, false)
|
|
141
|
+
when String then string(value, placeholders, prefix, key)
|
|
142
|
+
else raise Error, "cannot write a #{value.class} the way vcrpy does"
|
|
143
|
+
end
|
|
144
|
+
end
|
|
145
|
+
|
|
146
|
+
def string(value, placeholders, prefix, key)
|
|
147
|
+
if value.encoding == Encoding::BINARY
|
|
148
|
+
return Psych::Nodes::Scalar.new([value].pack("m57"), nil, "#{TAG}binary", false, false,
|
|
149
|
+
Psych::Nodes::Scalar::LITERAL)
|
|
150
|
+
end
|
|
151
|
+
return scalar(value, STR, resolve(value).nil?, true) if value.ascii_only?
|
|
152
|
+
|
|
153
|
+
token = "#{prefix}#{placeholders.size}z"
|
|
154
|
+
placeholders[token] = [value, key]
|
|
155
|
+
scalar(token, STR, true, true)
|
|
156
|
+
end
|
|
157
|
+
|
|
158
|
+
def scalar(value, tag, plain, quoted, style = Psych::Nodes::Scalar::ANY)
|
|
159
|
+
Psych::Nodes::Scalar.new(value, nil, tag, plain, quoted, style)
|
|
160
|
+
end
|
|
161
|
+
|
|
162
|
+
# PyYAML's represent_float.
|
|
163
|
+
def float_text(value)
|
|
164
|
+
return ".nan" if value.nan?
|
|
165
|
+
return value.positive? ? ".inf" : "-.inf" if value.infinite?
|
|
166
|
+
|
|
167
|
+
value.to_s
|
|
168
|
+
end
|
|
169
|
+
|
|
170
|
+
def splice(yaml, token, value, key)
|
|
171
|
+
index = yaml.index(token)
|
|
172
|
+
line_start = (yaml.rindex("\n", index) || -1) + 1
|
|
173
|
+
prefix = yaml[line_start...index]
|
|
174
|
+
quoted = double_quoted(value, column: prefix.length, indent: owner_column(prefix) + INDENT, allow_breaks: !key)
|
|
175
|
+
yaml.sub(token) { quoted }
|
|
176
|
+
end
|
|
177
|
+
|
|
178
|
+
# Column of the mapping key or sequence dash the scalar belongs to; libyaml indents continuation
|
|
179
|
+
# lines one step deeper than that.
|
|
180
|
+
def owner_column(prefix)
|
|
181
|
+
return prefix[/\A(?:\s*-\s)*\s*/].length if prefix.end_with?(": ")
|
|
182
|
+
|
|
183
|
+
prefix.rstrip.length - 1
|
|
184
|
+
end
|
|
185
|
+
|
|
186
|
+
# Port of yaml_emitter_write_double_quoted_scalar with unicode off. Keys are written without breaks.
|
|
187
|
+
def double_quoted(text, column:, indent:, allow_breaks: true)
|
|
188
|
+
out = +'"'
|
|
189
|
+
column += 1
|
|
190
|
+
spaces = false
|
|
191
|
+
chars = text.chars
|
|
192
|
+
chars.each_with_index do |char, index|
|
|
193
|
+
code = char.ord
|
|
194
|
+
if code < 0x20 || code > 0x7E || char == '"' || char == "\\"
|
|
195
|
+
escape = "\\#{ESCAPES[code] || hex_escape(code)}"
|
|
196
|
+
out << escape
|
|
197
|
+
column += escape.length
|
|
198
|
+
spaces = false
|
|
199
|
+
elsif char == " "
|
|
200
|
+
if allow_breaks && !spaces && column > BEST_WIDTH && index != 0 && index != chars.length - 1
|
|
201
|
+
out << "\n" << (" " * indent)
|
|
202
|
+
column = indent
|
|
203
|
+
if chars[index + 1] == " "
|
|
204
|
+
out << "\\"
|
|
205
|
+
column += 1
|
|
206
|
+
end
|
|
207
|
+
else
|
|
208
|
+
out << " "
|
|
209
|
+
column += 1
|
|
210
|
+
end
|
|
211
|
+
spaces = true
|
|
212
|
+
else
|
|
213
|
+
out << char
|
|
214
|
+
column += 1
|
|
215
|
+
spaces = false
|
|
216
|
+
end
|
|
217
|
+
end
|
|
218
|
+
out << '"'
|
|
219
|
+
end
|
|
220
|
+
|
|
221
|
+
def hex_escape(code)
|
|
222
|
+
if code <= 0xFF then format("x%02X", code)
|
|
223
|
+
elsif code <= 0xFFFF then format("u%04X", code)
|
|
224
|
+
else format("U%08X", code)
|
|
225
|
+
end
|
|
226
|
+
end
|
|
227
|
+
end
|
|
228
|
+
end
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module CassetteGuard
|
|
4
|
+
module Reporters
|
|
5
|
+
class Json
|
|
6
|
+
def initialize(config)
|
|
7
|
+
@config = config
|
|
8
|
+
end
|
|
9
|
+
|
|
10
|
+
def render(report, findings, hidden: 0)
|
|
11
|
+
document = {
|
|
12
|
+
version: VERSION,
|
|
13
|
+
cassettes: report.cassettes.size,
|
|
14
|
+
usage_checked: report.usage_checked,
|
|
15
|
+
hidden:,
|
|
16
|
+
occurrences: findings.sum(&:occurrences),
|
|
17
|
+
findings: findings.map(&:as_json),
|
|
18
|
+
skipped: report.skipped.map { |path, reason| { path:, reason: } },
|
|
19
|
+
}
|
|
20
|
+
"#{JSON.pretty_generate(document)}\n"
|
|
21
|
+
end
|
|
22
|
+
end
|
|
23
|
+
end
|
|
24
|
+
end
|
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module CassetteGuard
|
|
4
|
+
module Reporters
|
|
5
|
+
# SARIF 2.1.0, which GitHub code scanning turns into pull request annotations.
|
|
6
|
+
class Sarif
|
|
7
|
+
RULES = {
|
|
8
|
+
secret: ["Credential or token recorded in a cassette", "error"],
|
|
9
|
+
pii: ["Personal data recorded in a cassette", "error"],
|
|
10
|
+
size: ["Oversized cassette or body", "warning"],
|
|
11
|
+
unused: ["Cassette not loaded by any test", "warning"],
|
|
12
|
+
age: ["Cassette recorded long ago", "note"],
|
|
13
|
+
}.freeze
|
|
14
|
+
|
|
15
|
+
def initialize(config)
|
|
16
|
+
@config = config
|
|
17
|
+
end
|
|
18
|
+
|
|
19
|
+
def render(_report, findings, hidden: 0) # rubocop:disable Lint/UnusedMethodArgument
|
|
20
|
+
locators = Hash.new { |cache, cassette| cache[cassette] = LineLocator.new(read(cassette)) }
|
|
21
|
+
document = {
|
|
22
|
+
"$schema" => "https://json.schemastore.org/sarif-2.1.0.json",
|
|
23
|
+
"version" => "2.1.0",
|
|
24
|
+
"runs" => [{
|
|
25
|
+
"tool" => { "driver" => { "name" => "cassette_guard", "version" => VERSION, "rules" => rules } },
|
|
26
|
+
"results" => findings.map { |finding| result(finding, locators[finding.cassette]) },
|
|
27
|
+
}],
|
|
28
|
+
}
|
|
29
|
+
"#{JSON.pretty_generate(document)}\n"
|
|
30
|
+
end
|
|
31
|
+
|
|
32
|
+
private
|
|
33
|
+
|
|
34
|
+
def rules
|
|
35
|
+
RULES.map do |id, (description, level)|
|
|
36
|
+
{ "id" => id.to_s, "shortDescription" => { "text" => description }, "defaultConfiguration" => { "level" => level } }
|
|
37
|
+
end
|
|
38
|
+
end
|
|
39
|
+
|
|
40
|
+
def result(finding, locator)
|
|
41
|
+
{
|
|
42
|
+
"ruleId" => finding.rule.to_s,
|
|
43
|
+
"level" => RULES.fetch(finding.rule).last,
|
|
44
|
+
"message" => { "text" => message(finding) },
|
|
45
|
+
"locations" => [{
|
|
46
|
+
"physicalLocation" => {
|
|
47
|
+
"artifactLocation" => { "uri" => finding.cassette },
|
|
48
|
+
"region" => { "startLine" => locator.line_for(finding) },
|
|
49
|
+
},
|
|
50
|
+
}],
|
|
51
|
+
"partialFingerprints" => { "cassetteGuard/v1" => finding.fingerprint },
|
|
52
|
+
}
|
|
53
|
+
end
|
|
54
|
+
|
|
55
|
+
def message(finding)
|
|
56
|
+
text = [finding.where, finding.detail].reject(&:empty?).join(": ")
|
|
57
|
+
finding.occurrences > 1 ? "#{text} (#{finding.occurrences} interactions)" : text
|
|
58
|
+
end
|
|
59
|
+
|
|
60
|
+
def read(cassette)
|
|
61
|
+
File.read(File.expand_path(cassette, @config.root))
|
|
62
|
+
rescue SystemCallError
|
|
63
|
+
""
|
|
64
|
+
end
|
|
65
|
+
end
|
|
66
|
+
end
|
|
67
|
+
end
|
|
@@ -0,0 +1,76 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module CassetteGuard
|
|
4
|
+
module Reporters
|
|
5
|
+
class Text
|
|
6
|
+
LABELS = { secret: "SECRET", pii: "PII", size: "SIZE", unused: "UNUSED", age: "AGE" }.freeze
|
|
7
|
+
SHARED_MINIMUM = 3
|
|
8
|
+
SHARED_SHOWN = 5
|
|
9
|
+
|
|
10
|
+
def initialize(config)
|
|
11
|
+
@config = config
|
|
12
|
+
end
|
|
13
|
+
|
|
14
|
+
# findings are FindingGroups.
|
|
15
|
+
def render(report, findings, hidden: 0)
|
|
16
|
+
out = +""
|
|
17
|
+
findings.group_by(&:cassette).sort.each do |cassette, group|
|
|
18
|
+
out << cassette_block(cassette, group)
|
|
19
|
+
end
|
|
20
|
+
out << shared_secrets(findings)
|
|
21
|
+
out << summary(report, findings, hidden)
|
|
22
|
+
end
|
|
23
|
+
|
|
24
|
+
private
|
|
25
|
+
|
|
26
|
+
def cassette_block(cassette, group)
|
|
27
|
+
size = File.size?(File.expand_path(cassette, @config.root)).to_i
|
|
28
|
+
rows = group.sort_by { |finding| [finding.interaction || -1, finding.location.to_s, finding.kind] }
|
|
29
|
+
width = rows.map { |finding| finding.where.length }.max
|
|
30
|
+
lines = rows.map do |finding|
|
|
31
|
+
repeats = finding.occurrences > 1 ? " ×#{finding.occurrences}" : ""
|
|
32
|
+
format(" %-7s %-#{width}s %s%s", LABELS.fetch(finding.rule), finding.where, finding.detail, repeats).rstrip
|
|
33
|
+
end
|
|
34
|
+
"#{cassette} #{Format.bytes(size)}\n#{lines.join("\n")}\n\n"
|
|
35
|
+
end
|
|
36
|
+
|
|
37
|
+
# One leaked token recorded in hundreds of cassettes is one credential to rotate, not hundreds.
|
|
38
|
+
def shared_secrets(findings)
|
|
39
|
+
shared = findings.select { |finding| finding.rule == :secret }
|
|
40
|
+
.group_by { |finding| [finding.kind, finding.value] }
|
|
41
|
+
.transform_values { |group| group.map(&:cassette).uniq.size }
|
|
42
|
+
.select { |_, cassettes| cassettes >= SHARED_MINIMUM }
|
|
43
|
+
.sort_by { |(kind, _), cassettes| [-cassettes, kind] }
|
|
44
|
+
return "" if shared.empty?
|
|
45
|
+
|
|
46
|
+
rows = shared.first(SHARED_SHOWN).map do |(kind, value), cassettes|
|
|
47
|
+
" #{kind} #{Format.mask(value)} in #{Format.plural(cassettes, "cassette")}"
|
|
48
|
+
end
|
|
49
|
+
more = shared.size > SHARED_SHOWN ? "\n … and #{Format.plural(shared.size - SHARED_SHOWN, "more secret")}" : ""
|
|
50
|
+
"Secrets shared by several cassettes (rotate these first):\n#{rows.join("\n")}#{more}\n\n"
|
|
51
|
+
end
|
|
52
|
+
|
|
53
|
+
def summary(report, findings, hidden)
|
|
54
|
+
cassettes = Format.plural(report.cassettes.size, "cassette")
|
|
55
|
+
lines = []
|
|
56
|
+
if findings.empty?
|
|
57
|
+
lines << "No findings in #{cassettes}."
|
|
58
|
+
else
|
|
59
|
+
counts = LABELS.keys.filter_map do |rule|
|
|
60
|
+
count = findings.count { |finding| finding.rule == rule }
|
|
61
|
+
"#{count} #{rule}" if count.positive?
|
|
62
|
+
end
|
|
63
|
+
occurrences = findings.sum(&:occurrences)
|
|
64
|
+
places = occurrences > findings.size ? " at #{Format.number(occurrences)} places" : ""
|
|
65
|
+
lines << "#{cassettes} · #{Format.plural(findings.size, "finding")} (#{counts.join(", ")})#{places}"
|
|
66
|
+
end
|
|
67
|
+
lines << "#{Format.plural(hidden, "baselined finding")} hidden; --all shows them." if hidden.positive?
|
|
68
|
+
unless report.usage_checked
|
|
69
|
+
lines << "Unused cassettes not checked: no usage data in #{@config.usage_dir} (see README, \"Unused cassettes\")."
|
|
70
|
+
end
|
|
71
|
+
report.skipped.each { |path, reason| lines << "Skipped #{path}: #{reason}" }
|
|
72
|
+
"#{lines.join("\n")}\n"
|
|
73
|
+
end
|
|
74
|
+
end
|
|
75
|
+
end
|
|
76
|
+
end
|
|
@@ -0,0 +1,127 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module CassetteGuard
|
|
4
|
+
module Rules
|
|
5
|
+
SCRUB_PREFIX = "scrubbed-"
|
|
6
|
+
TRIM_PREFIX = "cassette_guard: trimmed"
|
|
7
|
+
|
|
8
|
+
SECRET_HEADERS = %w[
|
|
9
|
+
authorization proxy-authorization cookie set-cookie x-api-key api-key apikey
|
|
10
|
+
x-auth-token x-access-token x-amz-security-token x-goog-api-key
|
|
11
|
+
].freeze
|
|
12
|
+
|
|
13
|
+
SECRET_KEY_SUFFIXES = %w[
|
|
14
|
+
password passwd passphrase secret token apikey accesskey secretkey privatekey signature sessionid
|
|
15
|
+
].freeze
|
|
16
|
+
SECRET_KEYS = %w[auth pwd].freeze
|
|
17
|
+
NOT_SECRET_KEYS = %w[nexttoken nextpagetoken pagetoken continuationtoken synctoken csrftoken].freeze
|
|
18
|
+
|
|
19
|
+
PII_KEY_SUFFIXES = %w[
|
|
20
|
+
email emailaddress firstname lastname givenname familyname surname middlename fullname
|
|
21
|
+
birthdate dateofbirth birthday passport passportnumber passportno documentnumber idnumber
|
|
22
|
+
nationalid taxid socialsecuritynumber phone phonenumber mobilenumber mobilephone telephone
|
|
23
|
+
iban cardnumber streetaddress addressline1 addressline2
|
|
24
|
+
].freeze
|
|
25
|
+
PII_KEYS = %w[dob ssn pan mail mobile tel ccnumber street].freeze
|
|
26
|
+
CARD_KEYS = %w[pan ccnumber].freeze
|
|
27
|
+
|
|
28
|
+
TOKEN_PATTERNS = {
|
|
29
|
+
"jwt" => /\beyJ[A-Za-z0-9_-]{8,}\.eyJ[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}/,
|
|
30
|
+
"aws_access_key" => /\b(?:AKIA|ASIA)[0-9A-Z]{16}\b/,
|
|
31
|
+
"stripe_live_key" => /\b[rs]k_live_[0-9A-Za-z]{16,}\b/,
|
|
32
|
+
"github_token" => /\b(?:gh[pousr]_[A-Za-z0-9]{36,}|github_pat_[A-Za-z0-9_]{40,})\b/,
|
|
33
|
+
"slack_token" => /\bxox[abprs]-[A-Za-z0-9-]{10,}/,
|
|
34
|
+
"google_api_key" => /\bAIza[0-9A-Za-z_-]{35}\b/,
|
|
35
|
+
"private_key" => /-----BEGIN (?:[A-Z]+ )?PRIVATE KEY-----.*?-----END (?:[A-Z]+ )?PRIVATE KEY-----/m,
|
|
36
|
+
}.freeze
|
|
37
|
+
|
|
38
|
+
EMAIL = /\b[A-Za-z0-9._%+-]+@(?:[A-Za-z0-9-]+\.)+[A-Za-z]{2,}\b/
|
|
39
|
+
EMAIL_ONLY = /\A#{EMAIL.source}\z/
|
|
40
|
+
IBAN = /\b[A-Z]{2}\d{2}[A-Z0-9]{11,30}\b/
|
|
41
|
+
|
|
42
|
+
IBAN_LENGTHS = {
|
|
43
|
+
"AD" => 24, "AE" => 23, "AT" => 20, "BE" => 16, "BG" => 22, "CH" => 21, "CY" => 28, "CZ" => 24,
|
|
44
|
+
"DE" => 22, "DK" => 18, "EE" => 20, "ES" => 24, "FI" => 18, "FO" => 18, "FR" => 27, "GB" => 22,
|
|
45
|
+
"GI" => 23, "GL" => 18, "GR" => 27, "HR" => 21, "HU" => 28, "IE" => 22, "IL" => 23, "IS" => 26,
|
|
46
|
+
"IT" => 27, "LI" => 21, "LT" => 20, "LU" => 20, "LV" => 21, "MC" => 27, "MT" => 31, "NL" => 18,
|
|
47
|
+
"NO" => 15, "PL" => 28, "PT" => 25, "RO" => 24, "SA" => 24, "SE" => 24, "SI" => 19, "SK" => 24,
|
|
48
|
+
"SM" => 27, "TR" => 26,
|
|
49
|
+
}.freeze
|
|
50
|
+
|
|
51
|
+
ALLOWED_EMAIL_PATTERNS = %w[
|
|
52
|
+
*@example.com *@*.example.com *@example.org *@example.net *@*.example *@*.test *@*.invalid *@*.localhost
|
|
53
|
+
].freeze
|
|
54
|
+
|
|
55
|
+
TEST_CARD_NUMBERS = %w[
|
|
56
|
+
4111111111111111 4242424242424242 4000056655665556 4012888888881881 5555555555554444
|
|
57
|
+
5105105105105100 378282246310005 371449635398431 6011111111111117
|
|
58
|
+
].freeze
|
|
59
|
+
|
|
60
|
+
PLACEHOLDER_WORDS = %w[redacted filtered removed hidden scrubbed null nil none n/a na unknown].freeze
|
|
61
|
+
|
|
62
|
+
module_function
|
|
63
|
+
|
|
64
|
+
def normalize_key(key)
|
|
65
|
+
key.to_s.downcase.gsub(/[^a-z0-9]/, "")
|
|
66
|
+
end
|
|
67
|
+
|
|
68
|
+
def secret_key?(key)
|
|
69
|
+
normalized = normalize_key(key)
|
|
70
|
+
return false if normalized.empty? || NOT_SECRET_KEYS.include?(normalized)
|
|
71
|
+
|
|
72
|
+
SECRET_KEYS.include?(normalized) || SECRET_KEY_SUFFIXES.any? { |suffix| normalized.end_with?(suffix) }
|
|
73
|
+
end
|
|
74
|
+
|
|
75
|
+
def pii_key?(key)
|
|
76
|
+
normalized = normalize_key(key)
|
|
77
|
+
return false if normalized.empty?
|
|
78
|
+
|
|
79
|
+
PII_KEYS.include?(normalized) || PII_KEY_SUFFIXES.any? { |suffix| normalized.end_with?(suffix) }
|
|
80
|
+
end
|
|
81
|
+
|
|
82
|
+
def card_key?(key)
|
|
83
|
+
normalized = normalize_key(key)
|
|
84
|
+
CARD_KEYS.include?(normalized) || normalized.end_with?("cardnumber")
|
|
85
|
+
end
|
|
86
|
+
|
|
87
|
+
def placeholder?(value)
|
|
88
|
+
text = value.to_s.strip
|
|
89
|
+
text.empty? ||
|
|
90
|
+
text.start_with?(SCRUB_PREFIX, TRIM_PREFIX) ||
|
|
91
|
+
text.match?(/\A<[^<>]*>\z/) ||
|
|
92
|
+
text.match?(/\A\[[A-Z_ ]+\]\z/) ||
|
|
93
|
+
text.match?(/\A[*xX#._\- ]+\z/) ||
|
|
94
|
+
PLACEHOLDER_WORDS.include?(text.downcase)
|
|
95
|
+
end
|
|
96
|
+
|
|
97
|
+
def email?(value)
|
|
98
|
+
value.to_s.match?(EMAIL_ONLY)
|
|
99
|
+
end
|
|
100
|
+
|
|
101
|
+
def luhn?(value)
|
|
102
|
+
digits = value.to_s.delete(" -")
|
|
103
|
+
return false unless digits.match?(/\A\d{12,19}\z/)
|
|
104
|
+
|
|
105
|
+
sum = digits.reverse.each_char.with_index.sum do |char, index|
|
|
106
|
+
digit = char.to_i
|
|
107
|
+
next digit if index.even?
|
|
108
|
+
|
|
109
|
+
doubled = digit * 2
|
|
110
|
+
doubled > 9 ? doubled - 9 : doubled
|
|
111
|
+
end
|
|
112
|
+
(sum % 10).zero?
|
|
113
|
+
end
|
|
114
|
+
|
|
115
|
+
def iban?(value)
|
|
116
|
+
compact = value.to_s.delete(" ").upcase
|
|
117
|
+
return false unless compact.match?(/\A[A-Z]{2}\d{2}[A-Z0-9]+\z/)
|
|
118
|
+
return false unless IBAN_LENGTHS[compact[0, 2]] == compact.length
|
|
119
|
+
|
|
120
|
+
iban_remainder(compact[4..] + compact[0, 4]) == 1
|
|
121
|
+
end
|
|
122
|
+
|
|
123
|
+
def iban_remainder(text)
|
|
124
|
+
text.each_char.map { |char| char.match?(/\d/) ? char : (char.ord - 55).to_s }.join.to_i % 97
|
|
125
|
+
end
|
|
126
|
+
end
|
|
127
|
+
end
|
|
@@ -0,0 +1,131 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module CassetteGuard
|
|
4
|
+
class Scanner
|
|
5
|
+
Report = Data.define(:findings, :cassettes, :skipped, :usage_checked)
|
|
6
|
+
|
|
7
|
+
def initialize(config, now: Time.now)
|
|
8
|
+
@config = config
|
|
9
|
+
@secret = Detectors::Secret.new(config)
|
|
10
|
+
@pii = Detectors::Pii.new(config)
|
|
11
|
+
@size = Detectors::Size.new(config)
|
|
12
|
+
@age = Detectors::Age.new(config, now:)
|
|
13
|
+
end
|
|
14
|
+
|
|
15
|
+
def scan(paths = @config.cassette_files)
|
|
16
|
+
findings = []
|
|
17
|
+
cassettes = []
|
|
18
|
+
skipped = []
|
|
19
|
+
paths.each do |path|
|
|
20
|
+
cassette = Cassette.load(path)
|
|
21
|
+
if cassette.vcr?
|
|
22
|
+
cassettes << cassette.path
|
|
23
|
+
findings.concat(scan_cassette(cassette))
|
|
24
|
+
else
|
|
25
|
+
skipped << [@config.relative(path), "not a VCR cassette"]
|
|
26
|
+
end
|
|
27
|
+
rescue Error => e
|
|
28
|
+
skipped << [@config.relative(path), e.message]
|
|
29
|
+
end
|
|
30
|
+
usage = Detectors::Usage.new(@config)
|
|
31
|
+
findings.concat(usage.call(cassettes)) if usage.available?
|
|
32
|
+
Report.new(findings:, cassettes:, skipped:, usage_checked: usage.available?)
|
|
33
|
+
end
|
|
34
|
+
|
|
35
|
+
def scan_cassette(cassette)
|
|
36
|
+
relative_path = @config.relative(cassette.path)
|
|
37
|
+
findings = @size.file(cassette.path, relative_path) + @age.call(cassette, relative_path)
|
|
38
|
+
cassette.interactions.each_with_index do |interaction, index|
|
|
39
|
+
%w[request response].each do |side|
|
|
40
|
+
part = interaction[side]
|
|
41
|
+
findings.concat(PartScan.new(self, relative_path, index, side).call(part)) if part.is_a?(Hash)
|
|
42
|
+
end
|
|
43
|
+
end
|
|
44
|
+
findings
|
|
45
|
+
end
|
|
46
|
+
|
|
47
|
+
attr_reader :secret, :pii, :size
|
|
48
|
+
|
|
49
|
+
class PartScan
|
|
50
|
+
def initialize(scanner, cassette, interaction, side)
|
|
51
|
+
@scanner = scanner
|
|
52
|
+
@cassette = cassette
|
|
53
|
+
@interaction = interaction
|
|
54
|
+
@side = side
|
|
55
|
+
end
|
|
56
|
+
|
|
57
|
+
def call(part)
|
|
58
|
+
findings = []
|
|
59
|
+
findings.concat(uri(part["uri"])) if @side == "request" && part["uri"].is_a?(String)
|
|
60
|
+
findings.concat(headers(part["headers"])) if part["headers"].is_a?(Hash)
|
|
61
|
+
body = Body.for(part)
|
|
62
|
+
findings.concat(body(body)) if body
|
|
63
|
+
findings
|
|
64
|
+
end
|
|
65
|
+
|
|
66
|
+
private
|
|
67
|
+
|
|
68
|
+
def uri(uri)
|
|
69
|
+
parsed = parse_uri(uri)
|
|
70
|
+
fields = parsed ? uri_fields(parsed) : [Field.new(path: "", key: nil, value: uri)]
|
|
71
|
+
findings = fields.flat_map { |field| field_findings(field, "request.uri #{field.path}".strip) }
|
|
72
|
+
password = parsed&.password && URI.decode_uri_component(parsed.password)
|
|
73
|
+
findings + @scanner.secret.uri_password(password).map do |kind, value|
|
|
74
|
+
build(:secret, kind, value, "request.uri userinfo")
|
|
75
|
+
end
|
|
76
|
+
end
|
|
77
|
+
|
|
78
|
+
def headers(headers)
|
|
79
|
+
headers.flat_map do |name, values|
|
|
80
|
+
location = "#{@side}.headers.#{name}"
|
|
81
|
+
Array(values).flat_map do |value|
|
|
82
|
+
secrets = @scanner.secret.header(name, value.to_s).map { |kind, found| build(:secret, kind, found, location) }
|
|
83
|
+
pii = @scanner.pii.field(Field.new(path: "", key: nil, value: value.to_s))
|
|
84
|
+
.reject { |_kind, found| secrets.any? { |secret| secret.value.include?(found) } }
|
|
85
|
+
.map { |kind, found| build(:pii, kind, found, location) }
|
|
86
|
+
secrets + pii
|
|
87
|
+
end
|
|
88
|
+
end
|
|
89
|
+
end
|
|
90
|
+
|
|
91
|
+
def body(body)
|
|
92
|
+
location = "#{@side}.body"
|
|
93
|
+
findings = @scanner.size.body(body, @cassette, @interaction, location)
|
|
94
|
+
return findings if body.binary?
|
|
95
|
+
|
|
96
|
+
findings + Fields.from_text(body.text, body.content_type).flat_map do |field|
|
|
97
|
+
field_findings(field, "#{location} #{field.path}".strip)
|
|
98
|
+
end
|
|
99
|
+
end
|
|
100
|
+
|
|
101
|
+
def field_findings(field, location)
|
|
102
|
+
secrets = @scanner.secret.field(field).map { |kind, value| build(:secret, kind, value, location, field.key) }
|
|
103
|
+
pii = @scanner.pii.field(field)
|
|
104
|
+
.reject { |_kind, value| secrets.any? { |secret| secret.value.include?(value) } }
|
|
105
|
+
.map { |kind, value| build(:pii, kind, value, location, field.key) }
|
|
106
|
+
secrets + pii
|
|
107
|
+
end
|
|
108
|
+
|
|
109
|
+
def build(rule, kind, value, location, key = nil)
|
|
110
|
+
key = nil unless kind.start_with?("key:")
|
|
111
|
+
Finding.new(rule:, kind:, cassette: @cassette, interaction: @interaction, location:, value:, key:)
|
|
112
|
+
end
|
|
113
|
+
|
|
114
|
+
def parse_uri(uri)
|
|
115
|
+
URI.parse(uri)
|
|
116
|
+
rescue URI::Error
|
|
117
|
+
nil
|
|
118
|
+
end
|
|
119
|
+
|
|
120
|
+
def uri_fields(parsed)
|
|
121
|
+
fields = []
|
|
122
|
+
fields << Field.new(path: "path", key: nil, value: URI.decode_uri_component(parsed.path)) if parsed.path
|
|
123
|
+
return fields unless parsed.query
|
|
124
|
+
|
|
125
|
+
fields + URI.decode_www_form(parsed.query).map { |key, value| Field.new(path: "?#{key}", key:, value:) }
|
|
126
|
+
rescue ArgumentError
|
|
127
|
+
fields + [Field.new(path: "query", key: nil, value: parsed.query)]
|
|
128
|
+
end
|
|
129
|
+
end
|
|
130
|
+
end
|
|
131
|
+
end
|