vpndetection 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/LICENSE +21 -0
- data/README.md +171 -0
- data/lib/vpndetection/api/database_api.rb +351 -0
- data/lib/vpndetection/api/lookup_api.rb +85 -0
- data/lib/vpndetection/api_client.rb +397 -0
- data/lib/vpndetection/api_error.rb +58 -0
- data/lib/vpndetection/api_model_base.rb +88 -0
- data/lib/vpndetection/bogon.rb +60 -0
- data/lib/vpndetection/bogons.rb +67 -0
- data/lib/vpndetection/cache.rb +35 -0
- data/lib/vpndetection/client.rb +133 -0
- data/lib/vpndetection/configuration.rb +326 -0
- data/lib/vpndetection/database.rb +167 -0
- data/lib/vpndetection/errors.rb +107 -0
- data/lib/vpndetection/models/class_detail.rb +169 -0
- data/lib/vpndetection/models/dataset_checksums.rb +174 -0
- data/lib/vpndetection/models/dataset_checksums_response.rb +216 -0
- data/lib/vpndetection/models/dataset_format_size.rb +201 -0
- data/lib/vpndetection/models/dataset_list.rb +166 -0
- data/lib/vpndetection/models/dataset_metadata.rb +280 -0
- data/lib/vpndetection/models/dataset_metadata_column.rb +199 -0
- data/lib/vpndetection/models/download.rb +276 -0
- data/lib/vpndetection/models/download_list.rb +166 -0
- data/lib/vpndetection/models/error_envelope.rb +164 -0
- data/lib/vpndetection/models/licensed_dataset.rb +358 -0
- data/lib/vpndetection/models/licensed_version.rb +262 -0
- data/lib/vpndetection/models/lookup_error.rb +166 -0
- data/lib/vpndetection/models/lookup_response.rb +343 -0
- data/lib/vpndetection/models/proxy_detail.rb +199 -0
- data/lib/vpndetection/models/vpn_detail.rb +179 -0
- data/lib/vpndetection/result.rb +211 -0
- data/lib/vpndetection/retries.rb +33 -0
- data/lib/vpndetection/transport.rb +116 -0
- data/lib/vpndetection/version.rb +5 -0
- data/lib/vpndetection.rb +40 -0
- metadata +109 -0
|
@@ -0,0 +1,133 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require 'typhoeus'
|
|
4
|
+
|
|
5
|
+
module VPNDetection
|
|
6
|
+
DEFAULT_BASE_URL = 'https://api.vpndetection.io'
|
|
7
|
+
DEFAULT_CACHE_MAX_SIZE = 10_000
|
|
8
|
+
DEFAULT_CACHE_TTL = 3600
|
|
9
|
+
DEFAULT_CONCURRENCY = 8
|
|
10
|
+
DEFAULT_RETRIES = 2
|
|
11
|
+
DEFAULT_TIMEOUT = 10
|
|
12
|
+
|
|
13
|
+
# A client for the VPNDetection API.
|
|
14
|
+
#
|
|
15
|
+
# The cache is per instance, so an answer is never shared between two clients
|
|
16
|
+
# holding different API keys and therefore entitled to different fields.
|
|
17
|
+
class Client
|
|
18
|
+
# The licensed dataset downloads, for keys that carry the `db.download` scope.
|
|
19
|
+
attr_reader :database
|
|
20
|
+
|
|
21
|
+
# @param api_key [String, nil] omit it entirely to use the free tier, which
|
|
22
|
+
# answers `ip` and `is_vpn` and allows 1000 requests per day per source
|
|
23
|
+
# address.
|
|
24
|
+
# @param cache [Boolean] pass false to disable caching.
|
|
25
|
+
# @param cache_ttl [Numeric] how long an answer stays fresh, in seconds.
|
|
26
|
+
# @param concurrency [Integer] in-flight requests during a batch.
|
|
27
|
+
# @param retries [Integer] extra attempts for a transient failure.
|
|
28
|
+
# @param transport [Transport, nil] override the HTTP layer, mostly for tests.
|
|
29
|
+
def initialize(api_key: nil, base_url: DEFAULT_BASE_URL, cache: true,
|
|
30
|
+
cache_max_size: DEFAULT_CACHE_MAX_SIZE, cache_ttl: DEFAULT_CACHE_TTL,
|
|
31
|
+
concurrency: DEFAULT_CONCURRENCY, retries: DEFAULT_RETRIES,
|
|
32
|
+
timeout: DEFAULT_TIMEOUT, transport: nil)
|
|
33
|
+
@transport = transport || Transport.new(
|
|
34
|
+
Transport::Config.new(api_key: api_key, base_url: base_url, timeout: timeout),
|
|
35
|
+
)
|
|
36
|
+
@cache = cache ? Cache.new(max_size: cache_max_size, ttl: cache_ttl) : nil
|
|
37
|
+
@concurrency = concurrency
|
|
38
|
+
@retries = retries
|
|
39
|
+
@database = Database.new(@transport, retries: retries)
|
|
40
|
+
end
|
|
41
|
+
|
|
42
|
+
# Whether an address is private, loopback, link-local, documentation,
|
|
43
|
+
# multicast or otherwise not routable, including the IPv6 equivalents and
|
|
44
|
+
# the 6to4 and Teredo ranges.
|
|
45
|
+
#
|
|
46
|
+
# These are the addresses {#lookup} answers locally. Exposed here so the
|
|
47
|
+
# check is reachable from the client you already hold; the same predicate is
|
|
48
|
+
# also on the module itself, for code with no client to hand.
|
|
49
|
+
def bogon?(ip)
|
|
50
|
+
Bogon.bogon?(ip)
|
|
51
|
+
end
|
|
52
|
+
|
|
53
|
+
# Classify one address.
|
|
54
|
+
#
|
|
55
|
+
# A bogon is answered locally and never reaches the network. Everything else
|
|
56
|
+
# is served, then cached for this instance.
|
|
57
|
+
def lookup(ip, retries: nil)
|
|
58
|
+
return Bogon.result(ip) if Bogon.bogon?(ip)
|
|
59
|
+
|
|
60
|
+
hit = @cache&.get(ip)
|
|
61
|
+
return hit unless hit.nil?
|
|
62
|
+
|
|
63
|
+
result = Retries.with_retries(retries || @retries) do
|
|
64
|
+
Transport.lookup_result(@transport.lookup_request(ip).run)
|
|
65
|
+
end
|
|
66
|
+
@cache&.set(ip, result)
|
|
67
|
+
result
|
|
68
|
+
end
|
|
69
|
+
|
|
70
|
+
# Classify many addresses in parallel.
|
|
71
|
+
#
|
|
72
|
+
# Keyed by address rather than positional, so duplicates in the input
|
|
73
|
+
# collapse to a single request and the caller never has to line two lists
|
|
74
|
+
# up. An address that fails carries its error as its value, so one bad entry
|
|
75
|
+
# cannot lose the rest of the answers.
|
|
76
|
+
#
|
|
77
|
+
# @return [Hash{String => Result, Error}] in the order the addresses were given
|
|
78
|
+
def lookup_batch(ips, concurrency: nil, retries: nil)
|
|
79
|
+
addresses = ips.to_a.uniq
|
|
80
|
+
answers = {}
|
|
81
|
+
pending = []
|
|
82
|
+
|
|
83
|
+
addresses.each do |ip|
|
|
84
|
+
hit = Bogon.bogon?(ip) ? Bogon.result(ip) : @cache&.get(ip)
|
|
85
|
+
hit.nil? ? pending << ip : answers[ip] = hit
|
|
86
|
+
end
|
|
87
|
+
unless pending.empty?
|
|
88
|
+
run_batch(pending, answers, concurrency || @concurrency, retries || @retries)
|
|
89
|
+
end
|
|
90
|
+
|
|
91
|
+
# Reinstated in input order: a hydra settles in completion order, and a
|
|
92
|
+
# caller iterating the hash should see what they passed in.
|
|
93
|
+
addresses.to_h { |ip| [ip, answers[ip]] }
|
|
94
|
+
end
|
|
95
|
+
|
|
96
|
+
private
|
|
97
|
+
|
|
98
|
+
# One hydra per call, sized for THIS call. Reusing an instance-level hydra
|
|
99
|
+
# would silently cap a per-call concurrency at the client's setting, and
|
|
100
|
+
# would not be safe to drive from two threads either.
|
|
101
|
+
def run_batch(pending, answers, concurrency, retries)
|
|
102
|
+
hydra = Typhoeus::Hydra.new(max_concurrency: concurrency)
|
|
103
|
+
attempts = Hash.new(0)
|
|
104
|
+
|
|
105
|
+
enqueue = lambda do |ip|
|
|
106
|
+
request = @transport.lookup_request(ip)
|
|
107
|
+
request.on_complete do |response|
|
|
108
|
+
outcome = settle(ip, response, attempts, retries, enqueue)
|
|
109
|
+
answers[ip] = outcome unless outcome.nil?
|
|
110
|
+
end
|
|
111
|
+
hydra.queue(request)
|
|
112
|
+
end
|
|
113
|
+
|
|
114
|
+
pending.each { |ip| enqueue.call(ip) }
|
|
115
|
+
hydra.run
|
|
116
|
+
end
|
|
117
|
+
|
|
118
|
+
def settle(ip, response, attempts, retries, enqueue)
|
|
119
|
+
result = Transport.lookup_result(response)
|
|
120
|
+
@cache&.set(ip, result)
|
|
121
|
+
result
|
|
122
|
+
rescue Error => e
|
|
123
|
+
return e unless e.retryable? && attempts[ip] < retries
|
|
124
|
+
|
|
125
|
+
attempts[ip] += 1
|
|
126
|
+
# Sleeping here stalls the whole hydra, which is what a server-supplied
|
|
127
|
+
# delay asks for: it is telling every request to this host to back off.
|
|
128
|
+
sleep(Retries.delay_for(e, attempts[ip]))
|
|
129
|
+
enqueue.call(ip)
|
|
130
|
+
nil
|
|
131
|
+
end
|
|
132
|
+
end
|
|
133
|
+
end
|
|
@@ -0,0 +1,326 @@
|
|
|
1
|
+
=begin
|
|
2
|
+
#VPNDetection API
|
|
3
|
+
|
|
4
|
+
#The VPNDetection API: classify any IP address, and download the datasets behind the answers. See https://docs.vpndetection.io for guides and https://github.com/vpndetection-io for the official client libraries.
|
|
5
|
+
|
|
6
|
+
The version of the OpenAPI document: 2026.09.04
|
|
7
|
+
Contact: support@vpndetection.io
|
|
8
|
+
Generated by: https://openapi-generator.tech
|
|
9
|
+
Generator version: 7.25.0
|
|
10
|
+
|
|
11
|
+
=end
|
|
12
|
+
|
|
13
|
+
module VPNDetection
|
|
14
|
+
class Configuration
|
|
15
|
+
# Defines url scheme
|
|
16
|
+
attr_accessor :scheme
|
|
17
|
+
|
|
18
|
+
# Defines url host
|
|
19
|
+
attr_accessor :host
|
|
20
|
+
|
|
21
|
+
# Defines url base path
|
|
22
|
+
attr_accessor :base_path
|
|
23
|
+
|
|
24
|
+
# Define server configuration index
|
|
25
|
+
attr_accessor :server_index
|
|
26
|
+
|
|
27
|
+
# Define server operation configuration index
|
|
28
|
+
attr_accessor :server_operation_index
|
|
29
|
+
|
|
30
|
+
# Default server variables
|
|
31
|
+
attr_accessor :server_variables
|
|
32
|
+
|
|
33
|
+
# Default server operation variables
|
|
34
|
+
attr_accessor :server_operation_variables
|
|
35
|
+
|
|
36
|
+
# Defines API keys used with API Key authentications.
|
|
37
|
+
#
|
|
38
|
+
# @return [Hash] key: parameter name, value: parameter value (API key)
|
|
39
|
+
#
|
|
40
|
+
# @example parameter name is "api_key", API key is "xxx" (e.g. "api_key=xxx" in query string)
|
|
41
|
+
# config.api_key['api_key'] = 'xxx'
|
|
42
|
+
attr_accessor :api_key
|
|
43
|
+
|
|
44
|
+
# Defines API key prefixes used with API Key authentications.
|
|
45
|
+
#
|
|
46
|
+
# @return [Hash] key: parameter name, value: API key prefix
|
|
47
|
+
#
|
|
48
|
+
# @example parameter name is "Authorization", API key prefix is "Token" (e.g. "Authorization: Token xxx" in headers)
|
|
49
|
+
# config.api_key_prefix['api_key'] = 'Token'
|
|
50
|
+
attr_accessor :api_key_prefix
|
|
51
|
+
|
|
52
|
+
# Defines the username used with HTTP basic authentication.
|
|
53
|
+
#
|
|
54
|
+
# @return [String]
|
|
55
|
+
attr_accessor :username
|
|
56
|
+
|
|
57
|
+
# Defines the password used with HTTP basic authentication.
|
|
58
|
+
#
|
|
59
|
+
# @return [String]
|
|
60
|
+
attr_accessor :password
|
|
61
|
+
|
|
62
|
+
# Defines the access token (Bearer) used with OAuth2.
|
|
63
|
+
attr_accessor :access_token
|
|
64
|
+
|
|
65
|
+
# Defines a Proc used to fetch or refresh access tokens (Bearer) used with OAuth2.
|
|
66
|
+
# Overrides the access_token if set
|
|
67
|
+
# @return [Proc]
|
|
68
|
+
attr_accessor :access_token_getter
|
|
69
|
+
|
|
70
|
+
# Set this to return data as binary instead of downloading a temp file. When enabled (set to true)
|
|
71
|
+
# HTTP responses with return type `File` will be returned as a stream of binary data.
|
|
72
|
+
# Default to false.
|
|
73
|
+
attr_accessor :return_binary_data
|
|
74
|
+
|
|
75
|
+
# Set this to enable/disable debugging. When enabled (set to true), HTTP request/response
|
|
76
|
+
# details will be logged with `logger.debug` (see the `logger` attribute).
|
|
77
|
+
# Default to false.
|
|
78
|
+
#
|
|
79
|
+
# @return [true, false]
|
|
80
|
+
attr_accessor :debugging
|
|
81
|
+
|
|
82
|
+
# Set this to ignore operation servers for the API client. This is useful when you need to
|
|
83
|
+
# send requests to a different server than the one specified in the OpenAPI document.
|
|
84
|
+
# Will default to the base url defined in the spec but can be overridden by setting
|
|
85
|
+
# `scheme`, `host`, `base_path` directly.
|
|
86
|
+
# Default to false.
|
|
87
|
+
# @return [true, false]
|
|
88
|
+
attr_accessor :ignore_operation_servers
|
|
89
|
+
|
|
90
|
+
# Defines the logger used for debugging.
|
|
91
|
+
# Default to `Rails.logger` (when in Rails) or logging to STDOUT.
|
|
92
|
+
#
|
|
93
|
+
# @return [#debug]
|
|
94
|
+
attr_accessor :logger
|
|
95
|
+
|
|
96
|
+
# Defines the temporary folder to store downloaded files
|
|
97
|
+
# (for API endpoints that have file response).
|
|
98
|
+
# Default to use `Tempfile`.
|
|
99
|
+
#
|
|
100
|
+
# @return [String]
|
|
101
|
+
attr_accessor :temp_folder_path
|
|
102
|
+
|
|
103
|
+
# The time limit for HTTP request in seconds.
|
|
104
|
+
# Default to 0 (never times out).
|
|
105
|
+
attr_accessor :timeout
|
|
106
|
+
|
|
107
|
+
# Set this to false to skip client side validation in the operation.
|
|
108
|
+
# Default to true.
|
|
109
|
+
# @return [true, false]
|
|
110
|
+
attr_accessor :client_side_validation
|
|
111
|
+
|
|
112
|
+
### TLS/SSL setting
|
|
113
|
+
# Set this to false to skip verifying SSL certificate when calling API from https server.
|
|
114
|
+
# Default to true.
|
|
115
|
+
#
|
|
116
|
+
# @note Do NOT set it to false in production code, otherwise you would face multiple types of cryptographic attacks.
|
|
117
|
+
#
|
|
118
|
+
# @return [true, false]
|
|
119
|
+
attr_accessor :verify_ssl
|
|
120
|
+
|
|
121
|
+
### TLS/SSL setting
|
|
122
|
+
# Set this to false to skip verifying SSL host name
|
|
123
|
+
# Default to true.
|
|
124
|
+
#
|
|
125
|
+
# @note Do NOT set it to false in production code, otherwise you would face multiple types of cryptographic attacks.
|
|
126
|
+
#
|
|
127
|
+
# @return [true, false]
|
|
128
|
+
attr_accessor :verify_ssl_host
|
|
129
|
+
|
|
130
|
+
### TLS/SSL setting
|
|
131
|
+
# Set this to customize the certificate file to verify the peer.
|
|
132
|
+
#
|
|
133
|
+
# @return [String] the path to the certificate file
|
|
134
|
+
#
|
|
135
|
+
# @see The `cainfo` option of Typhoeus, `--cert` option of libcurl. Related source code:
|
|
136
|
+
# https://github.com/typhoeus/typhoeus/blob/master/lib/typhoeus/easy_factory.rb#L145
|
|
137
|
+
attr_accessor :ssl_ca_cert
|
|
138
|
+
|
|
139
|
+
### TLS/SSL setting
|
|
140
|
+
# Client certificate file (for client certificate)
|
|
141
|
+
attr_accessor :cert_file
|
|
142
|
+
|
|
143
|
+
### TLS/SSL setting
|
|
144
|
+
# Client private key file (for client certificate)
|
|
145
|
+
attr_accessor :key_file
|
|
146
|
+
|
|
147
|
+
# Set this to customize parameters encoding of array parameter with multi collectionFormat.
|
|
148
|
+
# Default to nil.
|
|
149
|
+
#
|
|
150
|
+
# @see The params_encoding option of Ethon. Related source code:
|
|
151
|
+
# https://github.com/typhoeus/ethon/blob/master/lib/ethon/easy/queryable.rb#L96
|
|
152
|
+
attr_accessor :params_encoding
|
|
153
|
+
|
|
154
|
+
|
|
155
|
+
attr_accessor :inject_format
|
|
156
|
+
|
|
157
|
+
attr_accessor :force_ending_format
|
|
158
|
+
|
|
159
|
+
def initialize
|
|
160
|
+
@scheme = 'https'
|
|
161
|
+
@host = 'api.vpndetection.io'
|
|
162
|
+
@base_path = ''
|
|
163
|
+
@server_index = nil
|
|
164
|
+
@server_operation_index = {}
|
|
165
|
+
@server_variables = {}
|
|
166
|
+
@server_operation_variables = {}
|
|
167
|
+
@api_key = {}
|
|
168
|
+
@api_key_prefix = {}
|
|
169
|
+
@client_side_validation = true
|
|
170
|
+
@verify_ssl = true
|
|
171
|
+
@verify_ssl_host = true
|
|
172
|
+
@cert_file = nil
|
|
173
|
+
@key_file = nil
|
|
174
|
+
@timeout = 0
|
|
175
|
+
@params_encoding = nil
|
|
176
|
+
@debugging = false
|
|
177
|
+
@ignore_operation_servers = false
|
|
178
|
+
@inject_format = false
|
|
179
|
+
@force_ending_format = false
|
|
180
|
+
@logger = defined?(Rails) ? Rails.logger : Logger.new(STDOUT)
|
|
181
|
+
|
|
182
|
+
yield(self) if block_given?
|
|
183
|
+
end
|
|
184
|
+
|
|
185
|
+
# The default Configuration object.
|
|
186
|
+
def self.default
|
|
187
|
+
@@default ||= Configuration.new
|
|
188
|
+
end
|
|
189
|
+
|
|
190
|
+
def configure
|
|
191
|
+
yield(self) if block_given?
|
|
192
|
+
end
|
|
193
|
+
|
|
194
|
+
def scheme=(scheme)
|
|
195
|
+
# remove :// from scheme
|
|
196
|
+
@scheme = scheme.sub(/:\/\//, '')
|
|
197
|
+
end
|
|
198
|
+
|
|
199
|
+
def host=(host)
|
|
200
|
+
# remove http(s):// and anything after a slash
|
|
201
|
+
@host = host.sub(/https?:\/\//, '').split('/').first
|
|
202
|
+
end
|
|
203
|
+
|
|
204
|
+
def base_path=(base_path)
|
|
205
|
+
# Add leading and trailing slashes to base_path
|
|
206
|
+
@base_path = "/#{base_path}".gsub(/\/+/, '/')
|
|
207
|
+
@base_path = '' if @base_path == '/'
|
|
208
|
+
end
|
|
209
|
+
|
|
210
|
+
# Returns base URL for specified operation based on server settings
|
|
211
|
+
def base_url(operation = nil)
|
|
212
|
+
return "#{scheme}://#{[host, base_path].join('/').gsub(/\/+/, '/')}".sub(/\/+\z/, '') if ignore_operation_servers
|
|
213
|
+
if operation_server_settings.key?(operation) then
|
|
214
|
+
index = server_operation_index.fetch(operation, server_index)
|
|
215
|
+
server_url(index.nil? ? 0 : index, server_operation_variables.fetch(operation, server_variables), operation_server_settings[operation])
|
|
216
|
+
else
|
|
217
|
+
server_index.nil? ? "#{scheme}://#{[host, base_path].join('/').gsub(/\/+/, '/')}".sub(/\/+\z/, '') : server_url(server_index, server_variables, nil)
|
|
218
|
+
end
|
|
219
|
+
end
|
|
220
|
+
|
|
221
|
+
# Gets API key (with prefix if set).
|
|
222
|
+
# @param [String] param_name the parameter name of API key auth
|
|
223
|
+
def api_key_with_prefix(param_name, param_alias = nil)
|
|
224
|
+
key = @api_key[param_name]
|
|
225
|
+
key = @api_key.fetch(param_alias, key) unless param_alias.nil?
|
|
226
|
+
if @api_key_prefix[param_name]
|
|
227
|
+
"#{@api_key_prefix[param_name]} #{key}"
|
|
228
|
+
else
|
|
229
|
+
key
|
|
230
|
+
end
|
|
231
|
+
end
|
|
232
|
+
|
|
233
|
+
# Gets access_token using access_token_getter or uses the static access_token
|
|
234
|
+
def access_token_with_refresh
|
|
235
|
+
return access_token if access_token_getter.nil?
|
|
236
|
+
access_token_getter.call
|
|
237
|
+
end
|
|
238
|
+
|
|
239
|
+
# Gets Basic Auth token string
|
|
240
|
+
def basic_auth_token
|
|
241
|
+
'Basic ' + ["#{username}:#{password}"].pack('m').delete("\r\n")
|
|
242
|
+
end
|
|
243
|
+
|
|
244
|
+
# Returns Auth Settings hash for api client.
|
|
245
|
+
def auth_settings
|
|
246
|
+
{
|
|
247
|
+
'bearerAuth' =>
|
|
248
|
+
{
|
|
249
|
+
type: 'bearer',
|
|
250
|
+
in: 'header',
|
|
251
|
+
key: 'Authorization',
|
|
252
|
+
value: "Bearer #{access_token_with_refresh}"
|
|
253
|
+
},
|
|
254
|
+
'apiKeyHeader' =>
|
|
255
|
+
{
|
|
256
|
+
type: 'api_key',
|
|
257
|
+
in: 'header',
|
|
258
|
+
key: 'X-Api-Key',
|
|
259
|
+
value: api_key_with_prefix('X-Api-Key')
|
|
260
|
+
},
|
|
261
|
+
'apiKeyQuery' =>
|
|
262
|
+
{
|
|
263
|
+
type: 'api_key',
|
|
264
|
+
in: 'query',
|
|
265
|
+
key: 'apikey',
|
|
266
|
+
value: api_key_with_prefix('apikey')
|
|
267
|
+
},
|
|
268
|
+
}
|
|
269
|
+
end
|
|
270
|
+
|
|
271
|
+
# Returns an array of Server setting
|
|
272
|
+
def server_settings
|
|
273
|
+
[
|
|
274
|
+
{
|
|
275
|
+
url: "https://api.vpndetection.io",
|
|
276
|
+
description: "Production",
|
|
277
|
+
},
|
|
278
|
+
{
|
|
279
|
+
url: "https://api-staging.vpndetection.io",
|
|
280
|
+
description: "Staging",
|
|
281
|
+
}
|
|
282
|
+
]
|
|
283
|
+
end
|
|
284
|
+
|
|
285
|
+
def operation_server_settings
|
|
286
|
+
{
|
|
287
|
+
}
|
|
288
|
+
end
|
|
289
|
+
|
|
290
|
+
# Returns URL based on server settings
|
|
291
|
+
#
|
|
292
|
+
# @param index array index of the server settings
|
|
293
|
+
# @param variables hash of variable and the corresponding value
|
|
294
|
+
def server_url(index, variables = {}, servers = nil)
|
|
295
|
+
servers = server_settings if servers == nil
|
|
296
|
+
|
|
297
|
+
# check array index out of bound
|
|
298
|
+
if (index.nil? || index < 0 || index >= servers.size)
|
|
299
|
+
fail ArgumentError, "Invalid index #{index} when selecting the server. Must not be nil and must be less than #{servers.size}"
|
|
300
|
+
end
|
|
301
|
+
|
|
302
|
+
server = servers[index]
|
|
303
|
+
url = server[:url]
|
|
304
|
+
|
|
305
|
+
return url unless server.key? :variables
|
|
306
|
+
|
|
307
|
+
# go through variable and assign a value
|
|
308
|
+
server[:variables].each do |name, variable|
|
|
309
|
+
if variables.key?(name)
|
|
310
|
+
if (!server[:variables][name].key?(:enum_values) || server[:variables][name][:enum_values].include?(variables[name]))
|
|
311
|
+
url.gsub! "{" + name.to_s + "}", variables[name]
|
|
312
|
+
else
|
|
313
|
+
fail ArgumentError, "The variable `#{name}` in the server URL has invalid value #{variables[name]}. Must be #{server[:variables][name][:enum_values]}."
|
|
314
|
+
end
|
|
315
|
+
else
|
|
316
|
+
# use default value
|
|
317
|
+
url.gsub! "{" + name.to_s + "}", server[:variables][name][:default_value]
|
|
318
|
+
end
|
|
319
|
+
end
|
|
320
|
+
|
|
321
|
+
url
|
|
322
|
+
end
|
|
323
|
+
|
|
324
|
+
|
|
325
|
+
end
|
|
326
|
+
end
|
|
@@ -0,0 +1,167 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module VPNDetection
|
|
4
|
+
# The licensed dataset downloads, reached as `client.database`.
|
|
5
|
+
#
|
|
6
|
+
# Access is granted by contract rather than self-serve, so every method here
|
|
7
|
+
# needs a key carrying the `db.download` scope.
|
|
8
|
+
class Database
|
|
9
|
+
def initialize(transport, retries:)
|
|
10
|
+
@transport = transport
|
|
11
|
+
@api = DatabaseApi.new(transport)
|
|
12
|
+
@retries = retries
|
|
13
|
+
end
|
|
14
|
+
|
|
15
|
+
# The dataset FAMILIES your organization is licensed to download.
|
|
16
|
+
#
|
|
17
|
+
# A license is held against the family, while a download names one version,
|
|
18
|
+
# so the ids {#download}, {#download_bytes}, {#download_url} and {#checksums}
|
|
19
|
+
# take come from each family's `versions`, not from the family itself.
|
|
20
|
+
def list
|
|
21
|
+
call { @api.list_databases.datasets }
|
|
22
|
+
end
|
|
23
|
+
|
|
24
|
+
# What is inside one dataset: schema, samples, row count and sizes.
|
|
25
|
+
def metadata(id)
|
|
26
|
+
call { @api.database_metadata(id) }
|
|
27
|
+
end
|
|
28
|
+
|
|
29
|
+
# The digests for one dataset file.
|
|
30
|
+
#
|
|
31
|
+
# Returns the whole set rather than one algorithm: which digests a dataset
|
|
32
|
+
# publishes is the API's choice, not ours, and the response nests them one
|
|
33
|
+
# level down under `checksums`.
|
|
34
|
+
def checksums(id, format)
|
|
35
|
+
call { @api.database_checksum(id, format).checksums }
|
|
36
|
+
end
|
|
37
|
+
|
|
38
|
+
# Your organization's recent download attempts, newest first.
|
|
39
|
+
def downloads(limit: nil)
|
|
40
|
+
call { @api.list_downloads(limit.nil? ? {} : { limit: limit }).downloads }
|
|
41
|
+
end
|
|
42
|
+
|
|
43
|
+
# The time-limited URL for one dataset file.
|
|
44
|
+
#
|
|
45
|
+
# The URL is returned rather than the bytes so the caller decides how to
|
|
46
|
+
# transfer a file that routinely runs to gigabytes; the link authorizes the
|
|
47
|
+
# START of a transfer, so one already running is not interrupted when it
|
|
48
|
+
# lapses.
|
|
49
|
+
def download_url(id, format)
|
|
50
|
+
call { redirect_location(id, format) }
|
|
51
|
+
end
|
|
52
|
+
|
|
53
|
+
# Download one dataset file to `path`, and return the bytes written.
|
|
54
|
+
#
|
|
55
|
+
# The bytes land in a neighboring `.part` file that is renamed on completion,
|
|
56
|
+
# so a transfer that dies half way leaves no truncated file that reads as a
|
|
57
|
+
# whole dataset, and a refresh that fails does not destroy the copy already
|
|
58
|
+
# there. Nothing beyond one chunk is ever held in memory, whatever the
|
|
59
|
+
# dataset weighs.
|
|
60
|
+
def download(id, format, path)
|
|
61
|
+
partial = "#{path}.part"
|
|
62
|
+
begin
|
|
63
|
+
url = download_url(id, format)
|
|
64
|
+
written = Retries.with_retries(@retries) do
|
|
65
|
+
# Reopened per attempt, so a retry restarts the file rather than
|
|
66
|
+
# appending a second copy of the body to a half-written one.
|
|
67
|
+
File.open(partial, 'wb') { |file| stream(url) { |chunk| file.write(chunk) } }
|
|
68
|
+
end
|
|
69
|
+
File.rename(partial, path)
|
|
70
|
+
rescue StandardError
|
|
71
|
+
File.delete(partial) if File.exist?(partial)
|
|
72
|
+
raise
|
|
73
|
+
end
|
|
74
|
+
written
|
|
75
|
+
end
|
|
76
|
+
|
|
77
|
+
# Download one dataset file and hand back its bytes.
|
|
78
|
+
#
|
|
79
|
+
# **This holds the entire file in memory**, and the catalog spans five orders
|
|
80
|
+
# of magnitude: `cdn_ip_v1` is 10 KB while `resproxy_ip_90d_v1` is 1.79 GB.
|
|
81
|
+
# Reach for it at the small end, where the bytes go straight into a parser,
|
|
82
|
+
# and use {#download} for anything you have not measured.
|
|
83
|
+
def download_bytes(id, format)
|
|
84
|
+
url = download_url(id, format)
|
|
85
|
+
Retries.with_retries(@retries) do
|
|
86
|
+
bytes = String.new(encoding: Encoding::BINARY)
|
|
87
|
+
stream(url) { |chunk| bytes << chunk }
|
|
88
|
+
bytes
|
|
89
|
+
end
|
|
90
|
+
end
|
|
91
|
+
|
|
92
|
+
private
|
|
93
|
+
|
|
94
|
+
# Runs one transfer of a presigned link, handing each chunk to the block, and
|
|
95
|
+
# returns the bytes that reached it.
|
|
96
|
+
#
|
|
97
|
+
# Typhoeus only streams when a request carries an `on_body` callback: with
|
|
98
|
+
# one set, Ethon's write callback passes the chunk on INSTEAD of appending it
|
|
99
|
+
# to `response.body`, so the ceiling on a transfer of any size is one chunk.
|
|
100
|
+
# The callback must therefore never answer `:unyielded`, which is the value
|
|
101
|
+
# that means "nobody took this" and puts the chunk back in the buffer.
|
|
102
|
+
def stream(url)
|
|
103
|
+
written = 0
|
|
104
|
+
served = nil
|
|
105
|
+
request = @transport.storage_request(url)
|
|
106
|
+
request.on_headers { |response| served = response.code }
|
|
107
|
+
request.on_body do |chunk, _response|
|
|
108
|
+
# An error page has no bounded size, so a refusal is aborted here rather
|
|
109
|
+
# than read and then classified.
|
|
110
|
+
next :abort unless served == 200
|
|
111
|
+
|
|
112
|
+
yield chunk
|
|
113
|
+
written += chunk.bytesize
|
|
114
|
+
end
|
|
115
|
+
|
|
116
|
+
settle(request.run, written)
|
|
117
|
+
end
|
|
118
|
+
|
|
119
|
+
def settle(response, written)
|
|
120
|
+
status = response.code.to_i
|
|
121
|
+
# No status at all means the transfer never reached HTTP: DNS, connect,
|
|
122
|
+
# TLS or a timeout, and curl's own reason is the only thing that says
|
|
123
|
+
# which. It is also what a :partial_file arrives as once a status IS in
|
|
124
|
+
# hand, which is where a transfer that died mid-flight fails rather than
|
|
125
|
+
# leaving a short file that reads as a whole dataset. The declared-length
|
|
126
|
+
# check the other bindings hand-roll is curl's job here.
|
|
127
|
+
raise Error.from_transport(response) if status.zero?
|
|
128
|
+
|
|
129
|
+
unless status == 200
|
|
130
|
+
raise Error.from_status(status, response.headers, nil,
|
|
131
|
+
message: "object storage refused the download link with status #{status}")
|
|
132
|
+
end
|
|
133
|
+
raise Error.from_transport(response) unless response.success?
|
|
134
|
+
|
|
135
|
+
written
|
|
136
|
+
end
|
|
137
|
+
|
|
138
|
+
# The 302 is this operation's SUCCESS case, but the generated client treats
|
|
139
|
+
# every non-2xx as a failure, so it arrives as an ApiError carrying the
|
|
140
|
+
# Location header.
|
|
141
|
+
def redirect_location(id, format)
|
|
142
|
+
@api.download_database(id, format)
|
|
143
|
+
raise Error.new(:server_error, 'expected a redirect to object storage')
|
|
144
|
+
rescue ApiError => e
|
|
145
|
+
raise unless e.code == 302
|
|
146
|
+
|
|
147
|
+
location = e.response_headers && e.response_headers['Location']
|
|
148
|
+
raise Error.new(:server_error, 'the redirect carried no Location header', status: 302) if location.nil?
|
|
149
|
+
|
|
150
|
+
location.is_a?(Array) ? location.last : location
|
|
151
|
+
end
|
|
152
|
+
|
|
153
|
+
def call(&block)
|
|
154
|
+
Retries.with_retries(@retries) do
|
|
155
|
+
block.call
|
|
156
|
+
rescue ApiError => e
|
|
157
|
+
raise error_for(e)
|
|
158
|
+
end
|
|
159
|
+
end
|
|
160
|
+
|
|
161
|
+
def error_for(api_error)
|
|
162
|
+
return Error.new(:network, api_error.message) if api_error.code.to_i.zero?
|
|
163
|
+
|
|
164
|
+
Error.from_status(api_error.code, api_error.response_headers, api_error.response_body)
|
|
165
|
+
end
|
|
166
|
+
end
|
|
167
|
+
end
|