zerobounce-sdk 3.0.0 → 3.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +184 -17
- data/README.md +26 -13
- data/lib/zerobounce/api_key.rb +238 -7
- data/lib/zerobounce/api_urls.rb +53 -0
- data/lib/zerobounce/base_request.rb +199 -82
- data/lib/zerobounce/client.rb +12 -4
- data/lib/zerobounce/configuration.rb +19 -3
- data/lib/zerobounce/error.rb +19 -7
- data/lib/zerobounce/get_file_helper.rb +40 -27
- data/lib/zerobounce/http_transport.rb +229 -0
- data/lib/zerobounce/priority3.rb +187 -46
- data/lib/zerobounce/request.rb +25 -42
- data/lib/zerobounce/response_error.rb +27 -4
- data/lib/zerobounce/response_shape.rb +190 -0
- data/lib/zerobounce/strict_bool.rb +23 -0
- data/lib/zerobounce/strict_json.rb +165 -0
- data/lib/zerobounce/version.rb +1 -1
- data/lib/zerobounce/warmup.rb +155 -69
- data/lib/zerobounce.rb +138 -97
- metadata +5 -15
data/lib/zerobounce/api_key.rb
CHANGED
|
@@ -1,21 +1,252 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
+
require 'erb'
|
|
4
|
+
require 'json'
|
|
5
|
+
require 'uri'
|
|
6
|
+
|
|
3
7
|
module Zerobounce
|
|
4
8
|
# Checks an API key before it is sent.
|
|
5
|
-
module ApiKey
|
|
9
|
+
module ApiKey # rubocop:disable Metrics/ModuleLength
|
|
6
10
|
module_function
|
|
7
11
|
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
12
|
+
CHARACTERS_PROBLEM = 'api_key has whitespace, control or invisible characters'
|
|
13
|
+
HEADER_PROBLEM = 'api_key has characters that cannot be sent in an HTTP header'
|
|
14
|
+
UNSAFE_CHARACTERS = /[[:space:][:cntrl:]\p{Cf}\p{Zs}\p{Zl}\p{Zp}\u180e]/
|
|
15
|
+
|
|
16
|
+
# The key, unchanged, or an ArgumentError when it holds whitespace of any
|
|
17
|
+
# kind (NBSP, U+0085, U+2028, U+3000), control or Unicode format characters
|
|
18
|
+
# (U+200B, U+FEFF, U+180E) anywhere, or anything outside printable ASCII
|
|
19
|
+
# (a key read from a file with its newline, or pasted with an invisible
|
|
20
|
+
# character). Such a key would fail at the API, and the HTTP library's own
|
|
11
21
|
# error could quote it unredacted.
|
|
12
22
|
def check!(api_key)
|
|
13
23
|
key = api_key.to_s
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
24
|
+
raise ArgumentError, HEADER_PROBLEM unless key.valid_encoding?
|
|
25
|
+
raise ArgumentError, CHARACTERS_PROBLEM if key.match?(UNSAFE_CHARACTERS)
|
|
26
|
+
raise ArgumentError, HEADER_PROBLEM if key.match?(/[^\x21-\x7e]/)
|
|
17
27
|
|
|
18
28
|
api_key
|
|
19
29
|
end
|
|
30
|
+
|
|
31
|
+
# Every form a secret can take in a body or URL echoed back to us,
|
|
32
|
+
# composed from the encoders up to depth 2: {raw, JSON-escaped,
|
|
33
|
+
# JSON-escaped with "\/"} (each JSON form with non-ASCII as is or as
|
|
34
|
+
# \uXXXX) x {as is, URL-encoded with %20, URL-encoded with +}, each JSON
|
|
35
|
+
# form escaped twice (a JSON error body quoting the JSON request body),
|
|
36
|
+
# and #inspect. Longest first, so a form that contains another is masked
|
|
37
|
+
# whole.
|
|
38
|
+
def secret_forms(secret)
|
|
39
|
+
raw = secret.to_s
|
|
40
|
+
return [] if raw.empty?
|
|
41
|
+
|
|
42
|
+
forms = escaped_forms(raw).flat_map { |form| url_forms(form) } + twice_escaped(raw) + [raw.inspect[1..-2]]
|
|
43
|
+
forms.uniq.reject(&:empty?).sort_by { |form| -form.length }
|
|
44
|
+
end
|
|
45
|
+
|
|
46
|
+
def escaped_forms(raw)
|
|
47
|
+
[raw] + json_escapers.map { |escape| escape.call(raw) }
|
|
48
|
+
end
|
|
49
|
+
|
|
50
|
+
def twice_escaped(raw)
|
|
51
|
+
json_escapers.map { |escape| escape.call(escape.call(raw)) }
|
|
52
|
+
end
|
|
53
|
+
|
|
54
|
+
def url_forms(raw)
|
|
55
|
+
[raw, ERB::Util.url_encode(raw), URI.encode_www_form_component(raw)]
|
|
56
|
+
end
|
|
57
|
+
|
|
58
|
+
# Text as it appears inside a JSON string: non-ASCII as is or as \uXXXX,
|
|
59
|
+
# "/" as is or as "\/".
|
|
60
|
+
def json_escapers
|
|
61
|
+
[->(text) { JSON.generate(text)[1..-2] },
|
|
62
|
+
->(text) { JSON.generate(text, ascii_only: true)[1..-2] },
|
|
63
|
+
->(text) { slash_escaped_json(text) },
|
|
64
|
+
->(text) { slash_escaped_json(text, ascii_only: true) }]
|
|
65
|
+
end
|
|
66
|
+
|
|
67
|
+
# The JSON string body with "/" written "\/", as some servers do. Built
|
|
68
|
+
# one character at a time with the JSON generator, so a backslash or quote
|
|
69
|
+
# in the secret is escaped exactly as JSON escapes it. (JSON's script_safe
|
|
70
|
+
# option does this too, but only from json 2.7; Ruby 3.0 to 3.2 ship older
|
|
71
|
+
# versions that ignore it.)
|
|
72
|
+
def slash_escaped_json(raw, **options)
|
|
73
|
+
raw.each_char.map { |char| char == '/' ? '\/' : JSON.generate(char, **options)[1..-2] }.join
|
|
74
|
+
end
|
|
75
|
+
|
|
76
|
+
# What an error says instead of text that still holds a secret in an
|
|
77
|
+
# encoding no spelling covers.
|
|
78
|
+
CREDENTIAL_OMITTED_MESSAGE =
|
|
79
|
+
'ZeroBounce returned an error; the response body was omitted because it contained a credential.'
|
|
80
|
+
|
|
81
|
+
# A secret shorter than this is masked only as a whole token (not touching
|
|
82
|
+
# a letter or digit), so a one-character password does not turn "Failed to
|
|
83
|
+
# open" into "Failed to o***en". From this length a secret is masked
|
|
84
|
+
# wherever it occurs; a password of 8 characters or more never appears in
|
|
85
|
+
# an ordinary word by chance.
|
|
86
|
+
WHOLE_TOKEN_BELOW = 8
|
|
87
|
+
|
|
88
|
+
# The text with every form of every secret replaced by ***, whatever the
|
|
89
|
+
# letter case. Then closed under encoding: if any decoded view of what is
|
|
90
|
+
# left (URL-decoding, "+" as a space, JSON unescaping incl. "\/" and
|
|
91
|
+
# \uXXXX, HTML entities, "\'" and "\"", up to four layers in any order)
|
|
92
|
+
# still holds a secret, the whole text is replaced by
|
|
93
|
+
# CREDENTIAL_OMITTED_MESSAGE.
|
|
94
|
+
def redact(text, secrets)
|
|
95
|
+
value = text.to_s.b
|
|
96
|
+
secrets = secrets.compact.map(&:to_s).reject { |secret| secret.strip.empty? }
|
|
97
|
+
secrets.each { |secret| value = mask(value, secret) }
|
|
98
|
+
value = value.force_encoding(Encoding::UTF_8).scrub
|
|
99
|
+
return CREDENTIAL_OMITTED_MESSAGE if holds_secret?(value, secrets)
|
|
100
|
+
|
|
101
|
+
value
|
|
102
|
+
end
|
|
103
|
+
|
|
104
|
+
# Masks one secret in the bytes: first as Unicode text when the bytes are
|
|
105
|
+
# valid UTF-8 (so letter case folds for non-ASCII letters too), then on
|
|
106
|
+
# the bytes themselves (so text that is not valid UTF-8 is covered).
|
|
107
|
+
def mask(bytes, secret)
|
|
108
|
+
forms = secret_forms(secret)
|
|
109
|
+
unicode = bytes.dup.force_encoding(Encoding::UTF_8)
|
|
110
|
+
if unicode.valid_encoding? && secret.valid_encoding?
|
|
111
|
+
bytes = unicode.gsub(secret_pattern(forms, secret, unicode: true), '***').b
|
|
112
|
+
end
|
|
113
|
+
bytes.gsub(secret_pattern(forms, secret), '***'.b)
|
|
114
|
+
end
|
|
115
|
+
|
|
116
|
+
# One case-insensitive pattern over the forms; unicode: true matches
|
|
117
|
+
# UTF-8 text with Unicode case folding, otherwise bytes. A short secret
|
|
118
|
+
# must stand as a whole token: the boundary applies at each end of the
|
|
119
|
+
# secret that is a letter or digit (a secret that starts or ends with
|
|
120
|
+
# punctuation is masked next to letters).
|
|
121
|
+
def secret_pattern(forms, secret, unicode: false) # rubocop:disable Metrics/CyclomaticComplexity
|
|
122
|
+
alnum = unicode ? '[[:alnum:]]' : '[A-Za-z0-9]'
|
|
123
|
+
body = forms.map { |form| Regexp.escape(unicode ? form : form.b) }.join('|')
|
|
124
|
+
if secret.length < WHOLE_TOKEN_BELOW
|
|
125
|
+
before = secret.match?(/\A[[:alnum:]]/) ? "(?<!#{alnum})" : ''
|
|
126
|
+
after = secret.match?(/[[:alnum:]]\z/) ? "(?!#{alnum})" : ''
|
|
127
|
+
body = "#{before}(?:#{body})#{after}"
|
|
128
|
+
end
|
|
129
|
+
unicode ? Regexp.new(body, Regexp::IGNORECASE) : Regexp.new(body.b, Regexp::IGNORECASE)
|
|
130
|
+
end
|
|
131
|
+
|
|
132
|
+
# Whether any decoded view of the text contains one of the secrets, in any
|
|
133
|
+
# letter case (Unicode-aware on valid UTF-8, and always on the bytes).
|
|
134
|
+
def holds_secret?(text, secrets)
|
|
135
|
+
return false if secrets.empty?
|
|
136
|
+
|
|
137
|
+
views = decoded_views(text)
|
|
138
|
+
secrets.any? { |secret| views.any? { |view| view_holds?(view, secret) } }
|
|
139
|
+
end
|
|
140
|
+
|
|
141
|
+
def view_holds?(view, secret)
|
|
142
|
+
forms = secret_forms(secret)
|
|
143
|
+
return true if view.match?(secret_pattern(forms, secret))
|
|
144
|
+
|
|
145
|
+
unicode = view.dup.force_encoding(Encoding::UTF_8)
|
|
146
|
+
unicode.valid_encoding? && secret.valid_encoding? && unicode.match?(secret_pattern(forms, secret, unicode: true))
|
|
147
|
+
end
|
|
148
|
+
|
|
149
|
+
# How many layers of encoding the check undoes: the SDK's own request
|
|
150
|
+
# encoding plus three echo layers (standard.md).
|
|
151
|
+
DECODE_LAYERS = 4
|
|
152
|
+
|
|
153
|
+
# The text (as bytes) and every view of it after up to DECODE_LAYERS decodings.
|
|
154
|
+
def decoded_views(text, layers = DECODE_LAYERS)
|
|
155
|
+
views = [text.to_s.b]
|
|
156
|
+
frontier = views
|
|
157
|
+
layers.times do
|
|
158
|
+
frontier = frontier.flat_map { |view| DECODERS.map { |decode| decode.call(view) } }.uniq - views
|
|
159
|
+
break if frontier.empty?
|
|
160
|
+
|
|
161
|
+
views += frontier
|
|
162
|
+
end
|
|
163
|
+
views
|
|
164
|
+
end
|
|
165
|
+
|
|
166
|
+
def url_decode(bytes)
|
|
167
|
+
bytes.gsub(/%([0-9A-Fa-f]{2})/n) { Regexp.last_match(1).hex.chr }
|
|
168
|
+
end
|
|
169
|
+
|
|
170
|
+
SURROGATE_PAIR = /\\u(d[89ab][0-9a-f]{2})\\u(d[c-f][0-9a-f]{2})/ni
|
|
171
|
+
JSON_ESCAPE = %r{\\(?:u([0-9a-fA-F]{4})|(["\\/bfnrt]))}n
|
|
172
|
+
SIMPLE_ESCAPES = { '"' => '"', '\\' => '\\', '/' => '/', 'b' => "\b", 'f' => "\f", 'n' => "\n",
|
|
173
|
+
'r' => "\r", 't' => "\t" }.freeze
|
|
174
|
+
|
|
175
|
+
def json_unescape(bytes) # rubocop:disable Metrics/AbcSize
|
|
176
|
+
paired = bytes.gsub(SURROGATE_PAIR) do
|
|
177
|
+
high = Regexp.last_match(1).hex
|
|
178
|
+
low = Regexp.last_match(2).hex
|
|
179
|
+
[0x10000 + ((high - 0xd800) << 10) + (low - 0xdc00)].pack('U').b
|
|
180
|
+
end
|
|
181
|
+
paired.gsub(JSON_ESCAPE) do
|
|
182
|
+
code = Regexp.last_match(1)
|
|
183
|
+
code ? [code.hex].pack('U').b : SIMPLE_ESCAPES.fetch(Regexp.last_match(2)).b
|
|
184
|
+
end
|
|
185
|
+
end
|
|
186
|
+
|
|
187
|
+
HTML_ENTITY = /&(?:#(\d{1,7});?|#[xX]([0-9a-fA-F]{1,6});?|(amp|lt|gt|quot|apos);)/n
|
|
188
|
+
NAMED_ENTITIES = { 'amp' => '&', 'lt' => '<', 'gt' => '>', 'quot' => '"', 'apos' => "'" }.freeze
|
|
189
|
+
|
|
190
|
+
# HTML entities decoded: d d ' " & ...
|
|
191
|
+
def html_unescape(bytes) # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/MethodLength, Metrics/PerceivedComplexity
|
|
192
|
+
bytes.gsub(HTML_ENTITY) do
|
|
193
|
+
match = Regexp.last_match
|
|
194
|
+
code = match[1]&.to_i || match[2]&.hex
|
|
195
|
+
if code.nil?
|
|
196
|
+
NAMED_ENTITIES.fetch(match[3]).b
|
|
197
|
+
elsif code.positive? && code <= 0x10FFFF && !code.between?(0xD800, 0xDFFF)
|
|
198
|
+
[code].pack('U').b
|
|
199
|
+
else
|
|
200
|
+
match[0]
|
|
201
|
+
end
|
|
202
|
+
end
|
|
203
|
+
end
|
|
204
|
+
|
|
205
|
+
def unescape_quotes(bytes)
|
|
206
|
+
bytes.gsub(/\\(['"])/n, '\1')
|
|
207
|
+
end
|
|
208
|
+
|
|
209
|
+
DECODERS = [
|
|
210
|
+
->(bytes) { url_decode(bytes) },
|
|
211
|
+
->(bytes) { url_decode(bytes.tr('+', ' ')) },
|
|
212
|
+
->(bytes) { bytes.tr('+', ' ') },
|
|
213
|
+
->(bytes) { json_unescape(bytes) },
|
|
214
|
+
->(bytes) { html_unescape(bytes) },
|
|
215
|
+
->(bytes) { unescape_quotes(bytes) }
|
|
216
|
+
].freeze
|
|
217
|
+
|
|
218
|
+
# Refuses Marshal and YAML serialisation of an object that holds an API
|
|
219
|
+
# key: the dump would carry the key to wherever it is written.
|
|
220
|
+
module RefusesDump
|
|
221
|
+
def marshal_dump
|
|
222
|
+
raise TypeError, "#{self.class.name} cannot be serialised: it holds an API key"
|
|
223
|
+
end
|
|
224
|
+
|
|
225
|
+
def _dump(_level)
|
|
226
|
+
marshal_dump
|
|
227
|
+
end
|
|
228
|
+
|
|
229
|
+
def encode_with(_coder)
|
|
230
|
+
marshal_dump
|
|
231
|
+
end
|
|
232
|
+
end
|
|
233
|
+
|
|
234
|
+
# The key a client holds. Its text is masked, so a dump of the client's
|
|
235
|
+
# instance variables (instance_variables, instance_variable_get, pp) shows
|
|
236
|
+
# [FILTERED] rather than the key. Read the key with #value.
|
|
237
|
+
class Secret
|
|
238
|
+
include RefusesDump
|
|
239
|
+
|
|
240
|
+
attr_reader :value
|
|
241
|
+
|
|
242
|
+
def initialize(value)
|
|
243
|
+
@value = value
|
|
244
|
+
end
|
|
245
|
+
|
|
246
|
+
def inspect
|
|
247
|
+
'[FILTERED]'
|
|
248
|
+
end
|
|
249
|
+
alias to_s inspect
|
|
250
|
+
end
|
|
20
251
|
end
|
|
21
252
|
end
|
data/lib/zerobounce/api_urls.rb
CHANGED
|
@@ -1,5 +1,8 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
+
require 'ipaddr'
|
|
4
|
+
require 'uri'
|
|
5
|
+
|
|
3
6
|
module Zerobounce
|
|
4
7
|
# API URL constants for Zerobounce endpoints
|
|
5
8
|
#
|
|
@@ -23,5 +26,55 @@ module Zerobounce
|
|
|
23
26
|
# URLs, not paths built from the bulk one.
|
|
24
27
|
LIST_EVALUATOR_URL = 'https://bulkapi.zerobounce.net/v2/listevaluator/'
|
|
25
28
|
INBOX_PLACEMENT_URL = 'https://bulkapi.zerobounce.net/api/ipt'
|
|
29
|
+
|
|
30
|
+
# Whitespace of any kind, control and Unicode format characters.
|
|
31
|
+
UNSAFE_URL_CHARACTERS = /[[:space:][:cntrl:]\p{Cf}\p{Zs}\p{Zl}\p{Zp}\u180e]/
|
|
32
|
+
LABEL = '[A-Za-z0-9](?:[A-Za-z0-9-]{0,61}[A-Za-z0-9])?'
|
|
33
|
+
HOST_LABELS = /\A#{LABEL}(?:\.#{LABEL})*\z/
|
|
34
|
+
private_constant :UNSAFE_URL_CHARACTERS, :LABEL, :HOST_LABELS
|
|
35
|
+
|
|
36
|
+
# The URL, unchanged, or an ArgumentError. Every request carries the API
|
|
37
|
+
# key, so it is refused unless the URL is an absolute https:// URL with a
|
|
38
|
+
# valid host and port, no userinfo (https://api.zerobounce.net@evil.example
|
|
39
|
+
# would send the key to evil.example), no whitespace, control or format
|
|
40
|
+
# characters and no backslash. Checked when a URL is configured and again
|
|
41
|
+
# on every send.
|
|
42
|
+
def self.check!(url, label = 'API URL')
|
|
43
|
+
text = url.to_s
|
|
44
|
+
raise ArgumentError, "#{label} must be an https:// URL" unless text.downcase.start_with?('https://')
|
|
45
|
+
raise ArgumentError, "#{label} must be a valid https:// URL" unless safe_https?(text)
|
|
46
|
+
|
|
47
|
+
url
|
|
48
|
+
end
|
|
49
|
+
|
|
50
|
+
def self.safe_https?(text) # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/MethodLength, Metrics/PerceivedComplexity
|
|
51
|
+
return false if text.match?(UNSAFE_URL_CHARACTERS) || text.include?('\\') || !text.valid_encoding?
|
|
52
|
+
|
|
53
|
+
authority = text[%r{\Ahttps://([^/?#]*)}i, 1]
|
|
54
|
+
return false if authority.nil? || authority.empty? || authority.include?('@') || authority.end_with?(':')
|
|
55
|
+
|
|
56
|
+
host, port = split_authority(authority)
|
|
57
|
+
return false if host.nil? || (port && !(1..65_535).cover?(port.to_i))
|
|
58
|
+
|
|
59
|
+
begin
|
|
60
|
+
URI.parse(text)
|
|
61
|
+
rescue URI::InvalidURIError
|
|
62
|
+
return false
|
|
63
|
+
end
|
|
64
|
+
host.start_with?('[') ? ipv6?(host) : host.match?(HOST_LABELS)
|
|
65
|
+
end
|
|
66
|
+
|
|
67
|
+
def self.split_authority(authority)
|
|
68
|
+
pattern = authority.start_with?('[') ? /\A(\[[^\]]*\])(?::(\d+))?\z/ : /\A([^:]+)(?::(\d+))?\z/
|
|
69
|
+
match = authority.match(pattern)
|
|
70
|
+
match && [match[1], match[2]]
|
|
71
|
+
end
|
|
72
|
+
|
|
73
|
+
def self.ipv6?(host)
|
|
74
|
+
IPAddr.new(host[1..-2].split('%', 2).first).ipv6?
|
|
75
|
+
rescue IPAddr::Error
|
|
76
|
+
false
|
|
77
|
+
end
|
|
78
|
+
private_class_method :safe_https?, :split_authority, :ipv6?
|
|
26
79
|
end
|
|
27
80
|
end
|