openasn 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.simplecov +25 -0
- data/AGENTS.md +23 -0
- data/CHANGELOG.md +52 -0
- data/LICENSE.txt +21 -0
- data/README.md +242 -0
- data/Rakefile +34 -0
- data/lib/generators/openasn/install/install_generator.rb +68 -0
- data/lib/generators/openasn/install/templates/openasn.rb +68 -0
- data/lib/openasn/binary_format.rb +305 -0
- data/lib/openasn/cidr_utils.rb +49 -0
- data/lib/openasn/classifier.rb +152 -0
- data/lib/openasn/configuration.rb +163 -0
- data/lib/openasn/data/seed/fetch-manifest.json +589 -0
- data/lib/openasn/data/seed/manifest.json +99 -0
- data/lib/openasn/data/seed/openasn-ipv4.bin +0 -0
- data/lib/openasn/data/seed/openasn-ipv6.bin +0 -0
- data/lib/openasn/dataset.rb +117 -0
- data/lib/openasn/errors.rb +22 -0
- data/lib/openasn/http_client.rb +86 -0
- data/lib/openasn/ip.rb +70 -0
- data/lib/openasn/middleware.rb +49 -0
- data/lib/openasn/overlay_store.rb +124 -0
- data/lib/openasn/parsers.rb +515 -0
- data/lib/openasn/railtie.rb +30 -0
- data/lib/openasn/result.rb +93 -0
- data/lib/openasn/snapshot.rb +171 -0
- data/lib/openasn/special_ranges.rb +44 -0
- data/lib/openasn/tier_b.rb +223 -0
- data/lib/openasn/update_job.rb +32 -0
- data/lib/openasn/updater.rb +173 -0
- data/lib/openasn/version.rb +5 -0
- data/lib/openasn.rb +115 -0
- metadata +85 -0
|
@@ -0,0 +1,515 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "json"
|
|
4
|
+
require "rexml/document"
|
|
5
|
+
require "zlib"
|
|
6
|
+
|
|
7
|
+
module OpenASN
|
|
8
|
+
# Tier B body parsers, keyed by the `parser` ids in fetch-manifest.json.
|
|
9
|
+
#
|
|
10
|
+
# Contract: parse(parser_id, body) -> Array of CIDR/IP string tokens
|
|
11
|
+
# (junk tokens are fine — CidrUtils drops them). UNKNOWN parser ids make
|
|
12
|
+
# #known? false and the executor SKIPS that source with a warning: that's
|
|
13
|
+
# the forward-compatibility deal that lets the data repo add sources
|
|
14
|
+
# without breaking old gems.
|
|
15
|
+
#
|
|
16
|
+
# Parsers are deliberately tolerant of cosmetic drift (extra columns,
|
|
17
|
+
# comments, header rows) and deliberately strict about shape drift (a
|
|
18
|
+
# JSON schema change raises ParseError -> the executor keeps stale data
|
|
19
|
+
# and records the error, visible in OpenASN.dataset_info).
|
|
20
|
+
module Parsers
|
|
21
|
+
class ParseError < Error; end
|
|
22
|
+
|
|
23
|
+
PARSERS = {}
|
|
24
|
+
|
|
25
|
+
def self.register(id, &block) = PARSERS[id] = block
|
|
26
|
+
def self.known?(id) = PARSERS.key?(id)
|
|
27
|
+
|
|
28
|
+
def self.parse(id, body)
|
|
29
|
+
handler = PARSERS[id] or raise ParseError, "unknown parser #{id}"
|
|
30
|
+
handler.call(body)
|
|
31
|
+
rescue ParseError
|
|
32
|
+
raise
|
|
33
|
+
rescue StandardError => e
|
|
34
|
+
raise ParseError, "#{id}: #{e.class}: #{e.message}"
|
|
35
|
+
end
|
|
36
|
+
|
|
37
|
+
# --- plain text shapes ---------------------------------------------------
|
|
38
|
+
|
|
39
|
+
register "plain_ip_per_line" do |body|
|
|
40
|
+
body.each_line.filter_map do |line|
|
|
41
|
+
t = line.strip
|
|
42
|
+
t unless t.empty? || t.start_with?("#")
|
|
43
|
+
end
|
|
44
|
+
end
|
|
45
|
+
|
|
46
|
+
register "plain_cidr_per_line" do |body|
|
|
47
|
+
body.each_line.filter_map do |line|
|
|
48
|
+
t = line.strip
|
|
49
|
+
t unless t.empty? || t.start_with?("#")
|
|
50
|
+
end
|
|
51
|
+
end
|
|
52
|
+
|
|
53
|
+
# First CSV column is a CIDR; used by Apple's relay list
|
|
54
|
+
# ("2.16.9.0/24,US,US-CA,,") and similar exports. Non-CIDR first
|
|
55
|
+
# columns (headers) simply fail CIDR parsing downstream and drop out.
|
|
56
|
+
register "csv_cidr_first_column" do |body|
|
|
57
|
+
body.each_line.filter_map do |line|
|
|
58
|
+
t = line.strip
|
|
59
|
+
next if t.empty? || t.start_with?("#")
|
|
60
|
+
|
|
61
|
+
t.split(",", 2).first&.strip
|
|
62
|
+
end
|
|
63
|
+
end
|
|
64
|
+
|
|
65
|
+
# RFC 8805 geofeeds: "prefix,country,region,city,zip" with '#' comments.
|
|
66
|
+
register "geofeed_csv" do |body|
|
|
67
|
+
body.each_line.filter_map do |line|
|
|
68
|
+
t = line.strip
|
|
69
|
+
next if t.empty? || t.start_with?("#")
|
|
70
|
+
|
|
71
|
+
t.split(",", 2).first&.strip
|
|
72
|
+
end
|
|
73
|
+
end
|
|
74
|
+
|
|
75
|
+
# --- structured cloud publications ---------------------------------------
|
|
76
|
+
|
|
77
|
+
register "aws_json" do |body|
|
|
78
|
+
data = JSON.parse(body)
|
|
79
|
+
v4 = (data["prefixes"] || []).filter_map { |p| p["ip_prefix"] }
|
|
80
|
+
v6 = (data["ipv6_prefixes"] || []).filter_map { |p| p["ipv6_prefix"] }
|
|
81
|
+
raise ParseError, "aws_json: no prefixes — schema changed?" if v4.empty? && v6.empty?
|
|
82
|
+
|
|
83
|
+
v4 + v6
|
|
84
|
+
end
|
|
85
|
+
|
|
86
|
+
register "gcp_json" do |body|
|
|
87
|
+
data = JSON.parse(body)
|
|
88
|
+
prefixes = (data["prefixes"] || []).filter_map { |p| p["ipv4Prefix"] || p["ipv6Prefix"] }
|
|
89
|
+
raise ParseError, "gcp_json: no prefixes — schema changed?" if prefixes.empty?
|
|
90
|
+
|
|
91
|
+
prefixes
|
|
92
|
+
end
|
|
93
|
+
|
|
94
|
+
register "azure_servicetags_json" do |body|
|
|
95
|
+
data = JSON.parse(body)
|
|
96
|
+
prefixes = (data["values"] || []).flat_map { |v| v.dig("properties", "addressPrefixes") || [] }
|
|
97
|
+
raise ParseError, "azure_servicetags_json: no addressPrefixes — schema changed?" if prefixes.empty?
|
|
98
|
+
|
|
99
|
+
prefixes
|
|
100
|
+
end
|
|
101
|
+
|
|
102
|
+
register "oci_json" do |body|
|
|
103
|
+
data = JSON.parse(body)
|
|
104
|
+
cidrs = (data["regions"] || []).flat_map { |r| (r["cidrs"] || []).filter_map { |c| c["cidr"] } }
|
|
105
|
+
raise ParseError, "oci_json: no cidrs — schema changed?" if cidrs.empty?
|
|
106
|
+
|
|
107
|
+
cidrs
|
|
108
|
+
end
|
|
109
|
+
|
|
110
|
+
# Zscaler CENR: nested {"zscaler.net": {"continent …": {"city …": [{"range": …}]}}}.
|
|
111
|
+
# Shape verified live 2026-07-04; we walk generically so cosmetic
|
|
112
|
+
# nesting changes don't break us.
|
|
113
|
+
register "zscaler_json" do |body|
|
|
114
|
+
ranges = []
|
|
115
|
+
walk = lambda do |node|
|
|
116
|
+
case node
|
|
117
|
+
when Hash
|
|
118
|
+
ranges << node["range"] if node["range"].is_a?(String)
|
|
119
|
+
node.each_value { |v| walk.call(v) }
|
|
120
|
+
when Array
|
|
121
|
+
node.each { |v| walk.call(v) }
|
|
122
|
+
end
|
|
123
|
+
end
|
|
124
|
+
walk.call(JSON.parse(body))
|
|
125
|
+
raise ParseError, "zscaler_json: no ranges — schema changed?" if ranges.empty?
|
|
126
|
+
|
|
127
|
+
ranges
|
|
128
|
+
end
|
|
129
|
+
|
|
130
|
+
# --- structured VPN provider publications -------------------------------
|
|
131
|
+
|
|
132
|
+
register "mullvad_relays_json" do |body|
|
|
133
|
+
data = JSON.parse(body)
|
|
134
|
+
raise ParseError, "mullvad_relays_json: expected array" unless data.is_a?(Array)
|
|
135
|
+
|
|
136
|
+
# First-party public API behind https://mullvad.net/en/servers. Mozilla
|
|
137
|
+
# VPN / Firefox VPN use Mullvad infrastructure, but the relay list cannot
|
|
138
|
+
# distinguish a Mozilla customer from a direct Mullvad customer, so the
|
|
139
|
+
# provider attribution remains the network operator: Mullvad.
|
|
140
|
+
tokens = data.select { |r| r["active"] != false }.flat_map do |relay|
|
|
141
|
+
[relay["ipv4_addr_in"], relay["ipv6_addr_in"]]
|
|
142
|
+
end.compact
|
|
143
|
+
raise ParseError, "mullvad_relays_json: no active relay IPs — schema changed?" if tokens.empty?
|
|
144
|
+
|
|
145
|
+
tokens.uniq
|
|
146
|
+
end
|
|
147
|
+
|
|
148
|
+
register "ivpn_servers_json" do |body|
|
|
149
|
+
data = JSON.parse(body)
|
|
150
|
+
tokens = []
|
|
151
|
+
(data["wireguard"] || []).each do |location|
|
|
152
|
+
(location["hosts"] || []).each { |host| tokens << host["host"] }
|
|
153
|
+
end
|
|
154
|
+
(data["openvpn"] || []).each do |location|
|
|
155
|
+
tokens.concat(location["ip_addresses"] || [])
|
|
156
|
+
end
|
|
157
|
+
tokens.compact!
|
|
158
|
+
raise ParseError, "ivpn_servers_json: no server IPs — schema changed?" if tokens.empty?
|
|
159
|
+
|
|
160
|
+
tokens.uniq
|
|
161
|
+
end
|
|
162
|
+
|
|
163
|
+
register "pia_servers_json" do |body|
|
|
164
|
+
# PIA appends a detached signature after the first JSON line. The first
|
|
165
|
+
# line is the server document the official clients consume.
|
|
166
|
+
data = JSON.parse(body.lines.first.to_s)
|
|
167
|
+
tokens = (data["regions"] || []).flat_map do |region|
|
|
168
|
+
next [] if region["offline"] == true
|
|
169
|
+
|
|
170
|
+
(region["servers"] || {}).values.flatten.filter_map { |server| server["ip"] }
|
|
171
|
+
end
|
|
172
|
+
raise ParseError, "pia_servers_json: no server IPs — schema changed?" if tokens.empty?
|
|
173
|
+
|
|
174
|
+
tokens.uniq
|
|
175
|
+
end
|
|
176
|
+
|
|
177
|
+
register "airvpn_status_json" do |body|
|
|
178
|
+
data = JSON.parse(body)
|
|
179
|
+
tokens = (data["servers"] || []).flat_map do |server|
|
|
180
|
+
server.filter_map do |key, value|
|
|
181
|
+
value if key.match?(/\Aip_v[46]_in\d+\z/)
|
|
182
|
+
end
|
|
183
|
+
end
|
|
184
|
+
raise ParseError, "airvpn_status_json: no entry IPs — schema changed?" if tokens.empty?
|
|
185
|
+
|
|
186
|
+
tokens.uniq
|
|
187
|
+
end
|
|
188
|
+
|
|
189
|
+
register "windscribe_serverlist_json" do |body|
|
|
190
|
+
data = JSON.parse(body)
|
|
191
|
+
tokens = (data["data"] || []).flat_map do |location|
|
|
192
|
+
next [] unless location["status"] == 1
|
|
193
|
+
|
|
194
|
+
(location["groups"] || []).flat_map do |group|
|
|
195
|
+
group_tokens = [group["ping_ip"]]
|
|
196
|
+
group_tokens.concat((group["nodes"] || []).flat_map { |node| [node["ip"], node["ip2"], node["ip3"]] })
|
|
197
|
+
group_tokens
|
|
198
|
+
end
|
|
199
|
+
end.compact
|
|
200
|
+
raise ParseError, "windscribe_serverlist_json: no server IPs — schema changed?" if tokens.empty?
|
|
201
|
+
|
|
202
|
+
tokens.uniq
|
|
203
|
+
end
|
|
204
|
+
|
|
205
|
+
register "privado_servers_json" do |body|
|
|
206
|
+
data = JSON.parse(body)
|
|
207
|
+
tokens = (data["servers"] || []).filter_map { |server| server["ip"] }
|
|
208
|
+
raise ParseError, "privado_servers_json: no server IPs — schema changed?" if tokens.empty?
|
|
209
|
+
|
|
210
|
+
tokens.uniq
|
|
211
|
+
end
|
|
212
|
+
|
|
213
|
+
register "leap_eip_service_json" do |body|
|
|
214
|
+
data = JSON.parse(body)
|
|
215
|
+
tokens = (data["gateways"] || []).filter_map { |gateway| gateway["ip_address"] }
|
|
216
|
+
raise ParseError, "leap_eip_service_json: no gateway IPs — schema changed?" if tokens.empty?
|
|
217
|
+
|
|
218
|
+
tokens.uniq
|
|
219
|
+
end
|
|
220
|
+
|
|
221
|
+
register "wlvpn_server_list_xml" do |body|
|
|
222
|
+
doc = REXML::Document.new(body)
|
|
223
|
+
tokens = []
|
|
224
|
+
doc.elements.each("//server") do |server|
|
|
225
|
+
next unless server.attributes["visible"].to_s == "1" && server.attributes["status"].to_s == "1"
|
|
226
|
+
|
|
227
|
+
ip = server.attributes["ip"].to_s.strip
|
|
228
|
+
tokens << ip unless ip.empty?
|
|
229
|
+
end
|
|
230
|
+
raise ParseError, "wlvpn_server_list_xml: no visible active server IPs — schema changed?" if tokens.empty?
|
|
231
|
+
|
|
232
|
+
tokens.uniq
|
|
233
|
+
end
|
|
234
|
+
|
|
235
|
+
register "surfshark_clusters_json" do |body|
|
|
236
|
+
data = JSON.parse(body)
|
|
237
|
+
raise ParseError, "surfshark_clusters_json: expected array" unless data.is_a?(Array)
|
|
238
|
+
|
|
239
|
+
tokens = data.filter_map { |cluster| cluster["connectionName"] }
|
|
240
|
+
raise ParseError, "surfshark_clusters_json: no connectionName hostnames — schema changed?" if tokens.empty?
|
|
241
|
+
|
|
242
|
+
tokens.uniq
|
|
243
|
+
end
|
|
244
|
+
|
|
245
|
+
register "nordvpn_servers_json" do |body|
|
|
246
|
+
data = JSON.parse(body)
|
|
247
|
+
data = data["servers"] if data.is_a?(Hash)
|
|
248
|
+
raise ParseError, "nordvpn_servers_json: expected array" unless data.is_a?(Array)
|
|
249
|
+
|
|
250
|
+
tokens = data.select { |server| server["status"] == "online" }.flat_map do |server|
|
|
251
|
+
ips = [server["station"], server["ipv6_station"], server["station_ipv6"]]
|
|
252
|
+
ips.concat((server["ips"] || []).filter_map { |entry| entry.dig("ip", "ip") })
|
|
253
|
+
ips
|
|
254
|
+
end.compact.reject(&:empty?)
|
|
255
|
+
raise ParseError, "nordvpn_servers_json: no server IPs — schema changed?" if tokens.empty?
|
|
256
|
+
|
|
257
|
+
tokens.uniq
|
|
258
|
+
end
|
|
259
|
+
|
|
260
|
+
register "vpngate_csv" do |body|
|
|
261
|
+
tokens = body.each_line.filter_map do |line|
|
|
262
|
+
next if line.start_with?("*", "#")
|
|
263
|
+
|
|
264
|
+
line.split(",", 3)[1]&.strip
|
|
265
|
+
end
|
|
266
|
+
raise ParseError, "vpngate_csv: no relay IPs — schema changed?" if tokens.empty?
|
|
267
|
+
|
|
268
|
+
tokens.uniq
|
|
269
|
+
end
|
|
270
|
+
|
|
271
|
+
register "ovpn_zip_remote_hosts" do |body|
|
|
272
|
+
tokens = unzip_files(body).flat_map do |name, content|
|
|
273
|
+
# TunnelBear's first-party Linux ZIP currently includes a few valid
|
|
274
|
+
# OpenVPN configs with a defensive ".ovpn.txt" suffix. Accept only
|
|
275
|
+
# the two explicit OpenVPN suffixes so README/license text cannot
|
|
276
|
+
# accidentally become source data.
|
|
277
|
+
next [] unless name.downcase.end_with?(".ovpn", ".ovpn.txt")
|
|
278
|
+
|
|
279
|
+
openvpn_remote_hosts(content)
|
|
280
|
+
end
|
|
281
|
+
raise ParseError, "ovpn_zip_remote_hosts: no OpenVPN remote hosts — schema changed?" if tokens.empty?
|
|
282
|
+
|
|
283
|
+
tokens.uniq
|
|
284
|
+
end
|
|
285
|
+
|
|
286
|
+
register "vpnbook_html_hosts" do |body|
|
|
287
|
+
tokens = body.scan(/\b[a-z0-9-]+\.vpnbook\.com\b/i).reject { |host| host.downcase == "www.vpnbook.com" }
|
|
288
|
+
raise ParseError, "vpnbook_html_hosts: no vpnbook.com hosts — schema changed?" if tokens.empty?
|
|
289
|
+
|
|
290
|
+
tokens.uniq
|
|
291
|
+
end
|
|
292
|
+
|
|
293
|
+
register "html_table_hostnames" do |body|
|
|
294
|
+
tokens = body.scan(%r{<td\b[^>]*>(.*?)</td>}im).flat_map do |cell|
|
|
295
|
+
# Some first-party support tables split hostnames across inline tags,
|
|
296
|
+
# e.g. `ca1.<span>vpn.giganews.com</span>`. Strip markup inside each
|
|
297
|
+
# cell before scanning so the parser stays generic without scraping
|
|
298
|
+
# arbitrary hostnames from the whole page chrome.
|
|
299
|
+
text = cell.first.to_s.dup.force_encoding(Encoding::UTF_8).scrub
|
|
300
|
+
.gsub(/<[^>]*>/, "").gsub(/ | /i, " ").tr("\u00A0", " ")
|
|
301
|
+
text.scan(/\b[a-z0-9.-]+\.[a-z]{2,63}\b/i)
|
|
302
|
+
end.map(&:downcase)
|
|
303
|
+
raise ParseError, "html_table_hostnames: no hostnames — schema changed?" if tokens.empty?
|
|
304
|
+
|
|
305
|
+
tokens.uniq
|
|
306
|
+
end
|
|
307
|
+
|
|
308
|
+
register "strongvpn_locations_html" do |body|
|
|
309
|
+
tokens = body.scan(/\bvpn-[a-z0-9-]+\.reliablehosting\.com\b/i).map(&:downcase)
|
|
310
|
+
raise ParseError, "strongvpn_locations_html: no StrongVPN hostnames — schema changed?" if tokens.empty?
|
|
311
|
+
|
|
312
|
+
tokens.uniq
|
|
313
|
+
end
|
|
314
|
+
|
|
315
|
+
register "vpnsecure_locations_html" do |body|
|
|
316
|
+
tokens = body.scan(%r{</div>\s*([a-z]{2,3}\d+)\s*<span[^>]*class=["'][^"']*\bstatus--up\b[^"']*["'][^>]*>\s*up\s*</span>}i)
|
|
317
|
+
.flatten
|
|
318
|
+
.map { |host| "#{host.downcase}.isponeder.com" }
|
|
319
|
+
raise ParseError, "vpnsecure_locations_html: no up hosts — schema changed?" if tokens.empty?
|
|
320
|
+
|
|
321
|
+
tokens.uniq
|
|
322
|
+
end
|
|
323
|
+
|
|
324
|
+
register "worldvpn_servers_html" do |body|
|
|
325
|
+
# WorldVPN's public server table gives exact IPs next to
|
|
326
|
+
# *.ocservvpn.com hostnames. Keep this parser table-shaped instead of
|
|
327
|
+
# doing a whole-page IP scrape; WordPress/CSS assets can contain
|
|
328
|
+
# version-looking strings that must not become VPN evidence.
|
|
329
|
+
tokens = body.scan(/<tr\b.*?<\/tr>/mi).filter_map do |row|
|
|
330
|
+
cells = row.scan(/<td\b[^>]*>(.*?)<\/td>/mi).flatten.map { |cell| strip_html(cell) }
|
|
331
|
+
ip = cells[1].to_s
|
|
332
|
+
host = cells[2].to_s.downcase
|
|
333
|
+
next unless ip.match?(/\A(?:\d{1,3}\.){3}\d{1,3}\z/) && host.match?(/\A[a-z]{2}\d+\.ocservvpn\.com\z/)
|
|
334
|
+
|
|
335
|
+
ip
|
|
336
|
+
end
|
|
337
|
+
raise ParseError, "worldvpn_servers_html: no server IPs — schema changed?" if tokens.empty?
|
|
338
|
+
|
|
339
|
+
tokens.uniq
|
|
340
|
+
end
|
|
341
|
+
|
|
342
|
+
register "ovpn_status_servers_json" do |body|
|
|
343
|
+
data = JSON.parse(body)
|
|
344
|
+
rows = data["data"]
|
|
345
|
+
raise ParseError, "ovpn_status_servers_json: expected data array" unless rows.is_a?(Array)
|
|
346
|
+
|
|
347
|
+
tokens = rows.filter_map do |server|
|
|
348
|
+
next if server["online"] == false
|
|
349
|
+
|
|
350
|
+
server["ip"]
|
|
351
|
+
end
|
|
352
|
+
raise ParseError, "ovpn_status_servers_json: no online server IPs — schema changed?" if tokens.empty?
|
|
353
|
+
|
|
354
|
+
tokens.uniq
|
|
355
|
+
end
|
|
356
|
+
|
|
357
|
+
register "anonine_status_json" do |body|
|
|
358
|
+
data = JSON.parse(body)
|
|
359
|
+
raise ParseError, "anonine_status_json: expected array" unless data.is_a?(Array)
|
|
360
|
+
|
|
361
|
+
tokens = data.flat_map do |row|
|
|
362
|
+
server_ips = (row["servers"] || []).flat_map { |server| server["ips"] || [] }
|
|
363
|
+
[row["primary_ip"], *server_ips]
|
|
364
|
+
end.compact
|
|
365
|
+
raise ParseError, "anonine_status_json: no server IPs — schema changed?" if tokens.empty?
|
|
366
|
+
|
|
367
|
+
tokens.uniq
|
|
368
|
+
end
|
|
369
|
+
|
|
370
|
+
register "azirevpn_locations_json" do |body|
|
|
371
|
+
data = JSON.parse(body)
|
|
372
|
+
rows = data["locations"]
|
|
373
|
+
raise ParseError, "azirevpn_locations_json: expected locations array" unless rows.is_a?(Array)
|
|
374
|
+
|
|
375
|
+
tokens = rows.filter_map { |location| location["pool"] }
|
|
376
|
+
raise ParseError, "azirevpn_locations_json: no pool hostnames — schema changed?" if tokens.empty?
|
|
377
|
+
|
|
378
|
+
tokens.uniq
|
|
379
|
+
end
|
|
380
|
+
|
|
381
|
+
register "vpnac_status_html" do |body|
|
|
382
|
+
tokens = body.scan(%r{<td\b[^>]*>\s*([a-z0-9][a-z0-9.-]*\.vpn\.ac)\s*</td>}i)
|
|
383
|
+
.flatten
|
|
384
|
+
.map(&:downcase)
|
|
385
|
+
raise ParseError, "vpnac_status_html: no status table hostnames — schema changed?" if tokens.empty?
|
|
386
|
+
|
|
387
|
+
tokens.uniq
|
|
388
|
+
end
|
|
389
|
+
|
|
390
|
+
register "trustzone_servers_html" do |body|
|
|
391
|
+
tokens = body.scan(/\b[a-z0-9]+(?:-[a-z0-9]+)*\.trust\.zone\b/i)
|
|
392
|
+
.map(&:downcase)
|
|
393
|
+
.reject { |host| host == "www.trust.zone" || host == "trust.zone" }
|
|
394
|
+
raise ParseError, "trustzone_servers_html: no Trust.Zone server hostnames — schema changed?" if tokens.empty?
|
|
395
|
+
|
|
396
|
+
tokens.uniq
|
|
397
|
+
end
|
|
398
|
+
|
|
399
|
+
register "slickvpn_locations_html" do |body|
|
|
400
|
+
tokens = body.scan(%r{<a\b[^>]*href=["']https://members\.newsdemon\.com/vpn/2025/[^"']+\.ovpn["'][^>]*>(.*?)</a>}im).flat_map do |label|
|
|
401
|
+
label.first.to_s.gsub(/<[^>]*>/, " ").scan(/\bgw\d+\.[a-z0-9.-]+\.slickvpn\.com\b/i)
|
|
402
|
+
end.map(&:downcase)
|
|
403
|
+
raise ParseError, "slickvpn_locations_html: no SlickVPN config-linked hostnames — schema changed?" if tokens.empty?
|
|
404
|
+
|
|
405
|
+
tokens.uniq
|
|
406
|
+
end
|
|
407
|
+
|
|
408
|
+
register "freevpn_us_status_html" do |body|
|
|
409
|
+
allowed = {
|
|
410
|
+
"openvpn" => /\Aovpn-[a-z0-9-]+\.vpnv\.cc\z/i,
|
|
411
|
+
"wireguard" => /\Awireguard-[a-z0-9-]+\.vpnv\.cc\z/i,
|
|
412
|
+
"pptp" => /\Apptp-[a-z0-9-]+\.vpnv\.cc\z/i
|
|
413
|
+
}
|
|
414
|
+
|
|
415
|
+
tokens = body.scan(/<tr\b[^>]*>/i).filter_map do |tag|
|
|
416
|
+
type = tag[/\bdata-type=["']([^"']+)["']/i, 1].to_s.downcase
|
|
417
|
+
host = tag[/\bdata-host=["']([^"']+)["']/i, 1].to_s.downcase
|
|
418
|
+
pattern = allowed[type]
|
|
419
|
+
host if pattern && host.match?(pattern)
|
|
420
|
+
end
|
|
421
|
+
raise ParseError, "freevpn_us_status_html: no VPN hosts — schema changed?" if tokens.empty?
|
|
422
|
+
|
|
423
|
+
tokens.uniq
|
|
424
|
+
end
|
|
425
|
+
|
|
426
|
+
class << self
|
|
427
|
+
private
|
|
428
|
+
|
|
429
|
+
def strip_html(fragment)
|
|
430
|
+
fragment.gsub(/<[^>]+>/, " ")
|
|
431
|
+
.gsub(/ | /i, " ")
|
|
432
|
+
.gsub(/\s+/, " ")
|
|
433
|
+
.strip
|
|
434
|
+
end
|
|
435
|
+
|
|
436
|
+
def openvpn_remote_hosts(content)
|
|
437
|
+
content.each_line.filter_map do |line|
|
|
438
|
+
match = line.match(/\Aremote\s+([^\s]+)(?:\s|$)/i)
|
|
439
|
+
match && match[1].delete_prefix("[").delete_suffix("]")
|
|
440
|
+
end
|
|
441
|
+
end
|
|
442
|
+
|
|
443
|
+
# Deflate expands up to ~1000:1 and these archives arrive from remote
|
|
444
|
+
# servers: cap inflated output so a hostile/compromised archive costs
|
|
445
|
+
# at most bounded memory (ParseError -> keep-stale), never an OOM.
|
|
446
|
+
# Real provider config archives inflate to single-digit MB.
|
|
447
|
+
MAX_INFLATED_BYTES = 64 * 1024 * 1024
|
|
448
|
+
|
|
449
|
+
# Minimal ZIP reader for first-party OpenVPN config archives. We keep
|
|
450
|
+
# this in stdlib Ruby instead of adding rubyzip so the gem stays
|
|
451
|
+
# dependency-free. It supports the two methods seen in provider archives:
|
|
452
|
+
# stored (0) and deflated (8), using the central directory so data
|
|
453
|
+
# descriptors in local file headers do not matter.
|
|
454
|
+
def unzip_files(body)
|
|
455
|
+
bytes = body.b
|
|
456
|
+
eocd = bytes.rindex("PK\x05\x06".b) or raise ParseError, "zip: missing end of central directory"
|
|
457
|
+
entries = bytes.byteslice(eocd + 10, 2).unpack1("v")
|
|
458
|
+
cd_offset = bytes.byteslice(eocd + 16, 4).unpack1("V")
|
|
459
|
+
pos = cd_offset
|
|
460
|
+
files = []
|
|
461
|
+
total_inflated = 0
|
|
462
|
+
|
|
463
|
+
entries.times do
|
|
464
|
+
raise ParseError, "zip: malformed central directory" unless bytes.byteslice(pos, 4) == "PK\x01\x02".b
|
|
465
|
+
|
|
466
|
+
method = bytes.byteslice(pos + 10, 2).unpack1("v")
|
|
467
|
+
compressed_size = bytes.byteslice(pos + 20, 4).unpack1("V")
|
|
468
|
+
name_length = bytes.byteslice(pos + 28, 2).unpack1("v")
|
|
469
|
+
extra_length = bytes.byteslice(pos + 30, 2).unpack1("v")
|
|
470
|
+
comment_length = bytes.byteslice(pos + 32, 2).unpack1("v")
|
|
471
|
+
local_offset = bytes.byteslice(pos + 42, 4).unpack1("V")
|
|
472
|
+
name = bytes.byteslice(pos + 46, name_length).force_encoding(Encoding::UTF_8).scrub
|
|
473
|
+
pos += 46 + name_length + extra_length + comment_length
|
|
474
|
+
|
|
475
|
+
next if name.end_with?("/")
|
|
476
|
+
raise ParseError, "zip: malformed local header for #{name}" unless bytes.byteslice(local_offset, 4) == "PK\x03\x04".b
|
|
477
|
+
|
|
478
|
+
local_name_length = bytes.byteslice(local_offset + 26, 2).unpack1("v")
|
|
479
|
+
local_extra_length = bytes.byteslice(local_offset + 28, 2).unpack1("v")
|
|
480
|
+
data_start = local_offset + 30 + local_name_length + local_extra_length
|
|
481
|
+
compressed = bytes.byteslice(data_start, compressed_size)
|
|
482
|
+
content = case method
|
|
483
|
+
when 0 then compressed
|
|
484
|
+
when 8 then bounded_inflate(compressed, name)
|
|
485
|
+
else
|
|
486
|
+
raise ParseError, "zip: unsupported compression method #{method} for #{name}"
|
|
487
|
+
end
|
|
488
|
+
total_inflated += content.bytesize
|
|
489
|
+
raise ParseError, "zip: archive inflates past #{MAX_INFLATED_BYTES} bytes - refusing" if total_inflated > MAX_INFLATED_BYTES
|
|
490
|
+
|
|
491
|
+
files << [name, content.force_encoding(Encoding::UTF_8).scrub]
|
|
492
|
+
end
|
|
493
|
+
|
|
494
|
+
files
|
|
495
|
+
end
|
|
496
|
+
|
|
497
|
+
# Inflate in chunks, aborting the moment cumulative output crosses the
|
|
498
|
+
# cap; the bomb never gets to materialize in memory.
|
|
499
|
+
def bounded_inflate(compressed, name)
|
|
500
|
+
inflater = Zlib::Inflate.new(-Zlib::MAX_WBITS)
|
|
501
|
+
out = +"".b
|
|
502
|
+
begin
|
|
503
|
+
inflater.inflate(compressed) do |chunk|
|
|
504
|
+
out << chunk
|
|
505
|
+
raise ParseError, "zip: #{name} inflates past #{MAX_INFLATED_BYTES} bytes - refusing" if out.bytesize > MAX_INFLATED_BYTES
|
|
506
|
+
end
|
|
507
|
+
out << inflater.finish unless inflater.finished?
|
|
508
|
+
ensure
|
|
509
|
+
inflater.close
|
|
510
|
+
end
|
|
511
|
+
out
|
|
512
|
+
end
|
|
513
|
+
end
|
|
514
|
+
end
|
|
515
|
+
end
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module OpenASN
|
|
4
|
+
class Railtie < Rails::Railtie
|
|
5
|
+
# Make OpenASN::UpdateJob exist as soon as ActiveJob does.
|
|
6
|
+
initializer "openasn.update_job" do
|
|
7
|
+
ActiveSupport.on_load(:active_job) { require "openasn/update_job" }
|
|
8
|
+
end
|
|
9
|
+
|
|
10
|
+
# Opportunistic staleness check: if the newest data on
|
|
11
|
+
# disk is older than a week and auto_update is on, enqueue a refresh.
|
|
12
|
+
# Uses a cheap file probe — it must NOT force the dataset to load at
|
|
13
|
+
# boot (lazy-load is the contract; eager_load! is opt-in). Deliberately
|
|
14
|
+
# skipped in test env, deliberately best-effort: a broken queue must
|
|
15
|
+
# never break boot.
|
|
16
|
+
config.after_initialize do
|
|
17
|
+
next if Rails.env.test?
|
|
18
|
+
next unless OpenASN.configuration.auto_update
|
|
19
|
+
|
|
20
|
+
begin
|
|
21
|
+
if OpenASN.data_stale_on_disk? && defined?(OpenASN::UpdateJob)
|
|
22
|
+
OpenASN::UpdateJob.perform_later
|
|
23
|
+
OpenASN.configuration.logger.info("openasn: dataset older than 7 days — background refresh enqueued")
|
|
24
|
+
end
|
|
25
|
+
rescue StandardError => e
|
|
26
|
+
OpenASN.configuration.logger.warn("openasn: boot staleness check skipped (#{e.message})")
|
|
27
|
+
end
|
|
28
|
+
end
|
|
29
|
+
end
|
|
30
|
+
end
|
|
@@ -0,0 +1,93 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module OpenASN
|
|
4
|
+
# The answer to "where is this IP really coming from?". Immutable.
|
|
5
|
+
#
|
|
6
|
+
# Verdict-first design: `verdict` is the closed enum; the predicates are
|
|
7
|
+
# sugar. There is deliberately NO `suspicious?` — that's a policy word,
|
|
8
|
+
# and drawing that line belongs to your application, not this gem.
|
|
9
|
+
#
|
|
10
|
+
# STABILITY CONTRACT (README "API stability contract" is the canonical
|
|
11
|
+
# text; keep in sync): VERDICTS is append-only — never remove, rename,
|
|
12
|
+
# or redefine an entry; additions land in minor versions with a loud
|
|
13
|
+
# CHANGELOG note. Verdicts are compiled code, never data: a data refresh
|
|
14
|
+
# cannot introduce one. to_h keys are append-only. The same contract
|
|
15
|
+
# binds every future client (openasn-js, …) — the enum is the project's
|
|
16
|
+
# cross-language API, defined in the data repo's DECISIONS.md.
|
|
17
|
+
class Result
|
|
18
|
+
VERDICTS = %i[
|
|
19
|
+
residential_isp mobile business hosting vpn tor_exit relay
|
|
20
|
+
enterprise_gateway education government cgnat private unknown
|
|
21
|
+
].freeze
|
|
22
|
+
|
|
23
|
+
# High-confidence "this is infrastructure, not an eyeball connection".
|
|
24
|
+
INFRASTRUCTURE_VERDICTS = %i[hosting vpn tor_exit].freeze
|
|
25
|
+
|
|
26
|
+
# "Very likely a human being on the other end" — including the classes
|
|
27
|
+
# people wrongly block: relay users are paying iCloud+ customers,
|
|
28
|
+
# CGNAT/mobile IPs are hundreds of people each, enterprise gateways are
|
|
29
|
+
# entire offices. Note the deliberate asymmetry: business/education/
|
|
30
|
+
# government/unknown are NEITHER infrastructure nor likely_human — your
|
|
31
|
+
# app decides those.
|
|
32
|
+
LIKELY_HUMAN_VERDICTS = %i[residential_isp mobile relay cgnat enterprise_gateway].freeze
|
|
33
|
+
|
|
34
|
+
attr_reader :ip, :verdict, :asn, :as_org, :category, :network_role,
|
|
35
|
+
:provider, :sources, :flags, :context_flags
|
|
36
|
+
|
|
37
|
+
def initialize(ip:, verdict:, asn: nil, as_org: nil, category: nil,
|
|
38
|
+
network_role: nil, provider: nil, sources: [], flags: 0,
|
|
39
|
+
context_flags: [], unrouted: false)
|
|
40
|
+
@ip = ip
|
|
41
|
+
@verdict = verdict
|
|
42
|
+
@asn = asn
|
|
43
|
+
@as_org = as_org
|
|
44
|
+
@category = category
|
|
45
|
+
@network_role = network_role
|
|
46
|
+
@provider = provider
|
|
47
|
+
@sources = sources.freeze
|
|
48
|
+
@flags = flags
|
|
49
|
+
@context_flags = context_flags.freeze
|
|
50
|
+
@unrouted = unrouted
|
|
51
|
+
freeze
|
|
52
|
+
end
|
|
53
|
+
|
|
54
|
+
def infrastructure? = INFRASTRUCTURE_VERDICTS.include?(verdict)
|
|
55
|
+
def likely_human? = LIKELY_HUMAN_VERDICTS.include?(verdict)
|
|
56
|
+
|
|
57
|
+
def vpn? = verdict == :vpn
|
|
58
|
+
def hosting? = verdict == :hosting
|
|
59
|
+
def tor? = verdict == :tor_exit
|
|
60
|
+
def relay? = verdict == :relay
|
|
61
|
+
def mobile? = verdict == :mobile
|
|
62
|
+
def private? = verdict == :private
|
|
63
|
+
def cgnat? = verdict == :cgnat
|
|
64
|
+
|
|
65
|
+
# True when no ASN announces this IP (unallocated/unrouted space).
|
|
66
|
+
def unrouted? = @unrouted
|
|
67
|
+
|
|
68
|
+
# Everything, for logging and shadow mode. Stable keys — CarHey-style
|
|
69
|
+
# shadow analyses depend on this shape staying append-only.
|
|
70
|
+
def to_h
|
|
71
|
+
{
|
|
72
|
+
ip: ip,
|
|
73
|
+
verdict: verdict,
|
|
74
|
+
infrastructure: infrastructure?,
|
|
75
|
+
likely_human: likely_human?,
|
|
76
|
+
asn: asn,
|
|
77
|
+
as_org: as_org,
|
|
78
|
+
category: category,
|
|
79
|
+
network_role: network_role,
|
|
80
|
+
provider: provider,
|
|
81
|
+
sources: sources,
|
|
82
|
+
flags: flags,
|
|
83
|
+
context_flags: context_flags,
|
|
84
|
+
unrouted: unrouted?
|
|
85
|
+
}
|
|
86
|
+
end
|
|
87
|
+
|
|
88
|
+
def inspect
|
|
89
|
+
"#<OpenASN::Result #{ip} verdict=#{verdict}#{asn ? " AS#{asn}" : ''}#{as_org ? " (#{as_org})" : ''} sources=#{sources.inspect}>"
|
|
90
|
+
end
|
|
91
|
+
alias to_s inspect
|
|
92
|
+
end
|
|
93
|
+
end
|