openasn 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.simplecov +25 -0
- data/AGENTS.md +23 -0
- data/CHANGELOG.md +52 -0
- data/LICENSE.txt +21 -0
- data/README.md +242 -0
- data/Rakefile +34 -0
- data/lib/generators/openasn/install/install_generator.rb +68 -0
- data/lib/generators/openasn/install/templates/openasn.rb +68 -0
- data/lib/openasn/binary_format.rb +305 -0
- data/lib/openasn/cidr_utils.rb +49 -0
- data/lib/openasn/classifier.rb +152 -0
- data/lib/openasn/configuration.rb +163 -0
- data/lib/openasn/data/seed/fetch-manifest.json +589 -0
- data/lib/openasn/data/seed/manifest.json +99 -0
- data/lib/openasn/data/seed/openasn-ipv4.bin +0 -0
- data/lib/openasn/data/seed/openasn-ipv6.bin +0 -0
- data/lib/openasn/dataset.rb +117 -0
- data/lib/openasn/errors.rb +22 -0
- data/lib/openasn/http_client.rb +86 -0
- data/lib/openasn/ip.rb +70 -0
- data/lib/openasn/middleware.rb +49 -0
- data/lib/openasn/overlay_store.rb +124 -0
- data/lib/openasn/parsers.rb +515 -0
- data/lib/openasn/railtie.rb +30 -0
- data/lib/openasn/result.rb +93 -0
- data/lib/openasn/snapshot.rb +171 -0
- data/lib/openasn/special_ranges.rb +44 -0
- data/lib/openasn/tier_b.rb +223 -0
- data/lib/openasn/update_job.rb +32 -0
- data/lib/openasn/updater.rb +173 -0
- data/lib/openasn/version.rb +5 -0
- data/lib/openasn.rb +115 -0
- metadata +85 -0
|
@@ -0,0 +1,305 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module OpenASN
|
|
4
|
+
# Reader for the OASN v1 artifact format and the OORG v1 org-names
|
|
5
|
+
# sidecar. The byte-exact spec lives in the data repo:
|
|
6
|
+
# https://github.com/openasn/openasn/blob/main/FORMAT.md — this file is an
|
|
7
|
+
# independent implementation of that document and must never drift from
|
|
8
|
+
# it. All integers big-endian. Readers REJECT unknown format versions on
|
|
9
|
+
# purpose: half-understanding an artifact is worse than keeping last-good
|
|
10
|
+
# data.
|
|
11
|
+
module BinaryFormat
|
|
12
|
+
MAGIC = "OASN"
|
|
13
|
+
ORG_MAGIC = "OORG"
|
|
14
|
+
FORMAT_VERSION = 0x01
|
|
15
|
+
HEADER_SIZE = 32
|
|
16
|
+
ORG_HEADER_SIZE = 16
|
|
17
|
+
|
|
18
|
+
# flags (u16) bit layout — see FORMAT.md
|
|
19
|
+
FLAG_BAD_ASN = 1 << 8
|
|
20
|
+
FLAG_VPN_PROVIDER = 1 << 9
|
|
21
|
+
FLAG_MOBILE = 1 << 10
|
|
22
|
+
FLAG_ENTERPRISE_GW = 1 << 11
|
|
23
|
+
FLAG_CDN = 1 << 12
|
|
24
|
+
FLAG_HOSTING_EXTRA = 1 << 13
|
|
25
|
+
|
|
26
|
+
CATEGORY_MASK = 0x000F
|
|
27
|
+
ROLE_SHIFT = 4
|
|
28
|
+
ROLE_MASK = 0x00F0
|
|
29
|
+
|
|
30
|
+
CATEGORIES = [nil, "isp", "hosting", "business", "education_research", "government_admin"].freeze
|
|
31
|
+
ROLES = [nil, "tier1_transit", "major_transit", "midsize_transit",
|
|
32
|
+
"access_provider", "content_network", "stub"].freeze
|
|
33
|
+
|
|
34
|
+
FLAG_NAMES = {
|
|
35
|
+
FLAG_BAD_ASN => :bad_asn,
|
|
36
|
+
FLAG_VPN_PROVIDER => :vpn_provider,
|
|
37
|
+
FLAG_MOBILE => :mobile_carrier,
|
|
38
|
+
FLAG_ENTERPRISE_GW => :enterprise_gw,
|
|
39
|
+
FLAG_CDN => :cdn,
|
|
40
|
+
FLAG_HOSTING_EXTRA => :hosting_extra
|
|
41
|
+
}.freeze
|
|
42
|
+
|
|
43
|
+
module_function
|
|
44
|
+
|
|
45
|
+
def category_name(flags) = CATEGORIES[flags & CATEGORY_MASK]
|
|
46
|
+
def role_name(flags) = ROLES[(flags & ROLE_MASK) >> ROLE_SHIFT]
|
|
47
|
+
|
|
48
|
+
def flag_names(flags)
|
|
49
|
+
FLAG_NAMES.filter_map { |bit, name| name if flags.anybits?(bit) }
|
|
50
|
+
end
|
|
51
|
+
|
|
52
|
+
def addr_size(family) = family == :ipv4 ? 4 : 16
|
|
53
|
+
def base_rec_size(family) = family == :ipv4 ? 14 : 38
|
|
54
|
+
def overlay_rec_size(family) = family == :ipv4 ? 8 : 32
|
|
55
|
+
|
|
56
|
+
def pack_addr(int, family)
|
|
57
|
+
if family == :ipv4
|
|
58
|
+
[int].pack("N")
|
|
59
|
+
else
|
|
60
|
+
[int >> 64, int & 0xFFFF_FFFF_FFFF_FFFF].pack("Q>Q>")
|
|
61
|
+
end
|
|
62
|
+
end
|
|
63
|
+
|
|
64
|
+
# Parse a full OASN artifact into its layers.
|
|
65
|
+
# -> { family:, build_ts:, base: BaseLayer, vpn: OverlayLayer, dc: OverlayLayer, relay: OverlayLayer }
|
|
66
|
+
def parse_artifact(bytes, mode: :packed)
|
|
67
|
+
raise FormatError, "not an OASN artifact (bad magic)" unless bytes[0, 4] == MAGIC
|
|
68
|
+
|
|
69
|
+
version = bytes.getbyte(4)
|
|
70
|
+
raise FormatError, "unsupported OASN format_version #{version} (this gem speaks v#{FORMAT_VERSION}); update the openasn gem" unless version == FORMAT_VERSION
|
|
71
|
+
|
|
72
|
+
family = bytes.getbyte(5) == 0x04 ? :ipv4 : :ipv6
|
|
73
|
+
build_ts = bytes[8, 8].unpack1("Q>")
|
|
74
|
+
base_n, vpn_n, dc_n, relay_n = bytes[16, 16].unpack("NNNN")
|
|
75
|
+
|
|
76
|
+
brec = base_rec_size(family)
|
|
77
|
+
orec = overlay_rec_size(family)
|
|
78
|
+
expected = HEADER_SIZE + base_n * brec + (vpn_n + dc_n + relay_n) * orec
|
|
79
|
+
raise FormatError, "artifact truncated or padded: #{bytes.bytesize} bytes, header implies #{expected}" unless bytes.bytesize == expected
|
|
80
|
+
|
|
81
|
+
offset = HEADER_SIZE
|
|
82
|
+
base = bytes[offset, base_n * brec]; offset += base_n * brec
|
|
83
|
+
vpn = bytes[offset, vpn_n * orec]; offset += vpn_n * orec
|
|
84
|
+
dc = bytes[offset, dc_n * orec]; offset += dc_n * orec
|
|
85
|
+
relay = bytes[offset, relay_n * orec]
|
|
86
|
+
|
|
87
|
+
{
|
|
88
|
+
family: family, build_ts: build_ts,
|
|
89
|
+
base: BaseLayer.build(base, family, mode),
|
|
90
|
+
vpn: OverlayLayer.build(vpn, family, mode),
|
|
91
|
+
dc: OverlayLayer.build(dc, family, mode),
|
|
92
|
+
relay: OverlayLayer.build(relay, family, mode)
|
|
93
|
+
}
|
|
94
|
+
end
|
|
95
|
+
|
|
96
|
+
# --- Base layer: [start, end, asn, flags] records -----------------------
|
|
97
|
+
|
|
98
|
+
module BaseLayer
|
|
99
|
+
def self.build(bytes, family, mode)
|
|
100
|
+
mode == :arrays ? ArraysBase.new(bytes, family) : PackedBase.new(bytes, family)
|
|
101
|
+
end
|
|
102
|
+
end
|
|
103
|
+
|
|
104
|
+
# Packed mode: binary search directly over the artifact bytes.
|
|
105
|
+
# ~6MB resident for all of IPv4, ~19µs/lookup (measured on the format's
|
|
106
|
+
# validation prototype). Key comparisons happen on raw big-endian
|
|
107
|
+
# address bytes: for unsigned BE values, bytewise String comparison IS
|
|
108
|
+
# numeric comparison, which lets IPv4 and IPv6 share one search.
|
|
109
|
+
class PackedBase
|
|
110
|
+
attr_reader :count
|
|
111
|
+
|
|
112
|
+
def initialize(bytes, family)
|
|
113
|
+
@bytes = bytes.freeze
|
|
114
|
+
@family = family
|
|
115
|
+
@asz = BinaryFormat.addr_size(family)
|
|
116
|
+
@rec = BinaryFormat.base_rec_size(family)
|
|
117
|
+
@count = bytes.bytesize / @rec
|
|
118
|
+
end
|
|
119
|
+
|
|
120
|
+
# -> [asn, flags] | nil
|
|
121
|
+
def find(ip_int)
|
|
122
|
+
key = BinaryFormat.pack_addr(ip_int, @family)
|
|
123
|
+
lo = 0
|
|
124
|
+
hi = @count - 1
|
|
125
|
+
while lo <= hi
|
|
126
|
+
mid = (lo + hi) / 2
|
|
127
|
+
off = mid * @rec
|
|
128
|
+
if key < @bytes[off, @asz]
|
|
129
|
+
hi = mid - 1
|
|
130
|
+
elsif key > @bytes[off + @asz, @asz]
|
|
131
|
+
lo = mid + 1
|
|
132
|
+
else
|
|
133
|
+
return @bytes[off + 2 * @asz, 6].unpack("Nn")
|
|
134
|
+
end
|
|
135
|
+
end
|
|
136
|
+
nil
|
|
137
|
+
end
|
|
138
|
+
end
|
|
139
|
+
|
|
140
|
+
# Arrays mode: unpack once into parallel Integer arrays. Measured on
|
|
141
|
+
# the real dataset: full lookups drop from ~15µs to ~9µs (the raw
|
|
142
|
+
# range probe alone is ~2µs, but IP parsing + overlay checks dominate
|
|
143
|
+
# the classify pipeline) at several× the memory. Worth it for
|
|
144
|
+
# lookup-heavy batch paths; configure via `memory_mode :arrays`.
|
|
145
|
+
class ArraysBase
|
|
146
|
+
attr_reader :count
|
|
147
|
+
|
|
148
|
+
def initialize(bytes, family)
|
|
149
|
+
asz = BinaryFormat.addr_size(family)
|
|
150
|
+
rec = BinaryFormat.base_rec_size(family)
|
|
151
|
+
@count = bytes.bytesize / rec
|
|
152
|
+
@starts = Array.new(@count)
|
|
153
|
+
@ends = Array.new(@count)
|
|
154
|
+
@asns = Array.new(@count)
|
|
155
|
+
@flags = Array.new(@count)
|
|
156
|
+
unpack_addr = family == :ipv4 ? ->(s) { s.unpack1("N") } : ->(s) { hi, lo = s.unpack("Q>Q>"); (hi << 64) | lo }
|
|
157
|
+
@count.times do |i|
|
|
158
|
+
off = i * rec
|
|
159
|
+
@starts[i] = unpack_addr.call(bytes[off, asz])
|
|
160
|
+
@ends[i] = unpack_addr.call(bytes[off + asz, asz])
|
|
161
|
+
@asns[i], @flags[i] = bytes[off + 2 * asz, 6].unpack("Nn")
|
|
162
|
+
end
|
|
163
|
+
[@starts, @ends, @asns, @flags].each(&:freeze)
|
|
164
|
+
end
|
|
165
|
+
|
|
166
|
+
def find(ip_int)
|
|
167
|
+
i = bsearch_le(@starts, ip_int)
|
|
168
|
+
return nil unless i && ip_int <= @ends[i]
|
|
169
|
+
|
|
170
|
+
[@asns[i], @flags[i]]
|
|
171
|
+
end
|
|
172
|
+
|
|
173
|
+
private
|
|
174
|
+
|
|
175
|
+
# Index of the greatest start <= ip_int (rightmost candidate range).
|
|
176
|
+
def bsearch_le(arr, val)
|
|
177
|
+
lo = 0
|
|
178
|
+
hi = arr.length - 1
|
|
179
|
+
ans = nil
|
|
180
|
+
while lo <= hi
|
|
181
|
+
mid = (lo + hi) / 2
|
|
182
|
+
if arr[mid] <= val
|
|
183
|
+
ans = mid
|
|
184
|
+
lo = mid + 1
|
|
185
|
+
else
|
|
186
|
+
hi = mid - 1
|
|
187
|
+
end
|
|
188
|
+
end
|
|
189
|
+
ans
|
|
190
|
+
end
|
|
191
|
+
end
|
|
192
|
+
|
|
193
|
+
# --- Overlay layers: sorted disjoint [start, end] ranges ----------------
|
|
194
|
+
|
|
195
|
+
module OverlayLayer
|
|
196
|
+
def self.build(bytes, family, mode)
|
|
197
|
+
mode == :arrays ? ArraysOverlay.new(bytes, family) : PackedOverlay.new(bytes, family)
|
|
198
|
+
end
|
|
199
|
+
end
|
|
200
|
+
|
|
201
|
+
class PackedOverlay
|
|
202
|
+
attr_reader :count
|
|
203
|
+
|
|
204
|
+
def initialize(bytes, family)
|
|
205
|
+
@bytes = bytes.freeze
|
|
206
|
+
@family = family
|
|
207
|
+
@asz = BinaryFormat.addr_size(family)
|
|
208
|
+
@rec = BinaryFormat.overlay_rec_size(family)
|
|
209
|
+
@count = bytes.bytesize / @rec
|
|
210
|
+
end
|
|
211
|
+
|
|
212
|
+
def cover?(ip_int)
|
|
213
|
+
return false if @count.zero?
|
|
214
|
+
|
|
215
|
+
key = BinaryFormat.pack_addr(ip_int, @family)
|
|
216
|
+
lo = 0
|
|
217
|
+
hi = @count - 1
|
|
218
|
+
while lo <= hi
|
|
219
|
+
mid = (lo + hi) / 2
|
|
220
|
+
off = mid * @rec
|
|
221
|
+
if key < @bytes[off, @asz]
|
|
222
|
+
hi = mid - 1
|
|
223
|
+
elsif key > @bytes[off + @asz, @asz]
|
|
224
|
+
lo = mid + 1
|
|
225
|
+
else
|
|
226
|
+
return true
|
|
227
|
+
end
|
|
228
|
+
end
|
|
229
|
+
false
|
|
230
|
+
end
|
|
231
|
+
end
|
|
232
|
+
|
|
233
|
+
class ArraysOverlay
|
|
234
|
+
attr_reader :count
|
|
235
|
+
|
|
236
|
+
def initialize(bytes, family)
|
|
237
|
+
asz = BinaryFormat.addr_size(family)
|
|
238
|
+
rec = BinaryFormat.overlay_rec_size(family)
|
|
239
|
+
@count = bytes.bytesize / rec
|
|
240
|
+
@starts = Array.new(@count)
|
|
241
|
+
@ends = Array.new(@count)
|
|
242
|
+
unpack_addr = family == :ipv4 ? ->(s) { s.unpack1("N") } : ->(s) { hi, lo = s.unpack("Q>Q>"); (hi << 64) | lo }
|
|
243
|
+
@count.times do |i|
|
|
244
|
+
off = i * rec
|
|
245
|
+
@starts[i] = unpack_addr.call(bytes[off, asz])
|
|
246
|
+
@ends[i] = unpack_addr.call(bytes[off + asz, asz])
|
|
247
|
+
end
|
|
248
|
+
[@starts, @ends].each(&:freeze)
|
|
249
|
+
end
|
|
250
|
+
|
|
251
|
+
def cover?(ip_int)
|
|
252
|
+
lo = 0
|
|
253
|
+
hi = @count - 1
|
|
254
|
+
while lo <= hi
|
|
255
|
+
mid = (lo + hi) / 2
|
|
256
|
+
if ip_int < @starts[mid]
|
|
257
|
+
hi = mid - 1
|
|
258
|
+
elsif ip_int > @ends[mid]
|
|
259
|
+
lo = mid + 1
|
|
260
|
+
else
|
|
261
|
+
return true
|
|
262
|
+
end
|
|
263
|
+
end
|
|
264
|
+
false
|
|
265
|
+
end
|
|
266
|
+
end
|
|
267
|
+
|
|
268
|
+
# --- OORG v1: ASN -> organization name ----------------------------------
|
|
269
|
+
|
|
270
|
+
class OrgIndex
|
|
271
|
+
def self.load(path)
|
|
272
|
+
new(File.binread(path))
|
|
273
|
+
end
|
|
274
|
+
|
|
275
|
+
def initialize(bytes)
|
|
276
|
+
raise FormatError, "not an OORG file (bad magic)" unless bytes[0, 4] == ORG_MAGIC
|
|
277
|
+
raise FormatError, "unsupported OORG version #{bytes.getbyte(4)}" unless bytes.getbyte(4) == 0x01
|
|
278
|
+
|
|
279
|
+
@bytes = bytes.freeze
|
|
280
|
+
@count, @blob_size = bytes[8, 8].unpack("NN")
|
|
281
|
+
@blob_base = ORG_HEADER_SIZE + @count * 8
|
|
282
|
+
expected = @blob_base + @blob_size
|
|
283
|
+
raise FormatError, "OORG truncated: #{bytes.bytesize} bytes, header implies #{expected}" unless bytes.bytesize == expected
|
|
284
|
+
end
|
|
285
|
+
|
|
286
|
+
def name(asn)
|
|
287
|
+
lo = 0
|
|
288
|
+
hi = @count - 1
|
|
289
|
+
while lo <= hi
|
|
290
|
+
mid = (lo + hi) / 2
|
|
291
|
+
a, off = @bytes[ORG_HEADER_SIZE + mid * 8, 8].unpack("NN")
|
|
292
|
+
if asn < a
|
|
293
|
+
hi = mid - 1
|
|
294
|
+
elsif asn > a
|
|
295
|
+
lo = mid + 1
|
|
296
|
+
else
|
|
297
|
+
nxt = mid + 1 < @count ? @bytes[ORG_HEADER_SIZE + (mid + 1) * 8 + 4, 4].unpack1("N") : @blob_size
|
|
298
|
+
return @bytes[@blob_base + off, nxt - off].force_encoding(Encoding::UTF_8)
|
|
299
|
+
end
|
|
300
|
+
end
|
|
301
|
+
nil
|
|
302
|
+
end
|
|
303
|
+
end
|
|
304
|
+
end
|
|
305
|
+
end
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "ipaddr"
|
|
4
|
+
|
|
5
|
+
module OpenASN
|
|
6
|
+
# Range math for the Tier B executor. Mirrors the data pipeline's
|
|
7
|
+
# semantics exactly (adjacent ranges merge; inputs may overlap freely).
|
|
8
|
+
module CidrUtils
|
|
9
|
+
module_function
|
|
10
|
+
|
|
11
|
+
# "1.2.3.0/24" | "1.2.3.4" -> [family, start_int, end_int] | nil (junk)
|
|
12
|
+
def parse(token)
|
|
13
|
+
ip = IPAddr.new(token.strip)
|
|
14
|
+
r = ip.to_range
|
|
15
|
+
[ip.ipv4? ? :ipv4 : :ipv6, r.first.to_i, r.last.to_i]
|
|
16
|
+
rescue IPAddr::Error
|
|
17
|
+
nil
|
|
18
|
+
end
|
|
19
|
+
|
|
20
|
+
# Merge overlapping AND adjacent ranges. Critical for Apple's relay
|
|
21
|
+
# list: ~280k rows collapse dramatically once merged, which is the
|
|
22
|
+
# difference between a fat linear file and a lookup-friendly overlay.
|
|
23
|
+
def merge(ranges)
|
|
24
|
+
return [] if ranges.empty?
|
|
25
|
+
|
|
26
|
+
sorted = ranges.sort_by { |r| [r[0], r[1]] }
|
|
27
|
+
merged = [[sorted[0][0], sorted[0][1]]]
|
|
28
|
+
sorted.each do |(s, e)|
|
|
29
|
+
last = merged.last
|
|
30
|
+
if s <= last[1] + 1
|
|
31
|
+
last[1] = e if e > last[1]
|
|
32
|
+
else
|
|
33
|
+
merged << [s, e]
|
|
34
|
+
end
|
|
35
|
+
end
|
|
36
|
+
merged
|
|
37
|
+
end
|
|
38
|
+
|
|
39
|
+
# tokens (CIDRs/IPs, junk tolerated) -> { ipv4: merged, ipv6: merged }
|
|
40
|
+
def ranges_by_family(tokens)
|
|
41
|
+
out = { ipv4: [], ipv6: [] }
|
|
42
|
+
tokens.each do |token|
|
|
43
|
+
family, s, e = parse(token)
|
|
44
|
+
out[family] << [s, e] if family
|
|
45
|
+
end
|
|
46
|
+
{ ipv4: merge(out[:ipv4]), ipv6: merge(out[:ipv6]) }
|
|
47
|
+
end
|
|
48
|
+
end
|
|
49
|
+
end
|
|
@@ -0,0 +1,152 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module OpenASN
|
|
4
|
+
# The verdict precedence ladder. First match wins. This ordering IS the
|
|
5
|
+
# product — change it only with the data repo's DECISIONS.md open next to
|
|
6
|
+
# you, because every line encodes a documented false-positive lesson:
|
|
7
|
+
#
|
|
8
|
+
# 1. invalid input -> InvalidIPError (raised in IP.parse)
|
|
9
|
+
# 2. special ranges -> :private / :cgnat
|
|
10
|
+
# 3. relay overlays (Tier B: Apple) -> :relay
|
|
11
|
+
# 4. tor overlays (Tier B) -> :tor_exit
|
|
12
|
+
# 5. vpn ranges (canonical X4B ∪ Tier B provider lists) -> :vpn
|
|
13
|
+
# 6. ASN flag vpn_provider -> :vpn
|
|
14
|
+
# 7. ASN flag enterprise_gw ∪ Tier B gateway ranges -> :enterprise_gateway
|
|
15
|
+
# 8. cloud provider ranges (Tier B) -> :hosting (provider tagged)
|
|
16
|
+
# 9. canonical dc overlay (X4B) -> :hosting
|
|
17
|
+
# 10. flags bad_asn|hosting_extra|cdn or category==hosting -> :hosting
|
|
18
|
+
# 11. ASN flag mobile_carrier -> :mobile
|
|
19
|
+
# 12. category isp, role != tier1_transit -> :residential_isp
|
|
20
|
+
# 13. category business -> :business
|
|
21
|
+
# 14. category education_research -> :education
|
|
22
|
+
# 15. category government_admin -> :government
|
|
23
|
+
# 16. category isp + tier1_transit -> :unknown (pure-backbone ambiguity)
|
|
24
|
+
# 17. ASN found, no category -> :unknown
|
|
25
|
+
# 18. no ASN -> :unknown (unrouted)
|
|
26
|
+
#
|
|
27
|
+
# Why relay outranks EVERYTHING data-driven: iCloud Private Relay egress
|
|
28
|
+
# lives inside Cloudflare/Akamai space, which the base layer correctly
|
|
29
|
+
# calls hosting — paying iCloud+ customers would classify as datacenter
|
|
30
|
+
# traffic without rule 3. Same defensive logic for enterprise gateways
|
|
31
|
+
# (rule 7 beats the hosting rules: Zscaler ranges are offices, not
|
|
32
|
+
# servers). Rules 12/16: see the data repo's DECISIONS.md D-IMPL-1 —
|
|
33
|
+
# every national telco carries a transit role in upstream data; only
|
|
34
|
+
# pure tier-1 backbone is genuinely ambiguous.
|
|
35
|
+
module Classifier
|
|
36
|
+
module_function
|
|
37
|
+
|
|
38
|
+
def classify(snapshot, ip_input)
|
|
39
|
+
family, ip_int = IP.parse(ip_input)
|
|
40
|
+
ip_string = ip_input.is_a?(String) ? ip_input : ip_input.to_s
|
|
41
|
+
|
|
42
|
+
# Rule 2 — specials don't need data at all.
|
|
43
|
+
if (special = SpecialRanges.match(ip_int, family))
|
|
44
|
+
return Result.new(ip: ip_string, verdict: special[0], sources: [special[1]])
|
|
45
|
+
end
|
|
46
|
+
|
|
47
|
+
layers = snapshot.family(family)
|
|
48
|
+
|
|
49
|
+
# The base layer is consulted regardless of the winning rule: even a
|
|
50
|
+
# Tor exit's Result should say which ASN announces it.
|
|
51
|
+
asn, flags = layers.base.find(ip_int)
|
|
52
|
+
flags ||= 0
|
|
53
|
+
category = BinaryFormat.category_name(flags)
|
|
54
|
+
role = BinaryFormat.role_name(flags)
|
|
55
|
+
|
|
56
|
+
# Context flags never decide verdicts; they ride along for app logic.
|
|
57
|
+
context = []
|
|
58
|
+
snapshot.overlays_for(family, "flag:cloudflare_range").each do |(_, layer)|
|
|
59
|
+
context << :cloudflare_range if layer.cover?(ip_int)
|
|
60
|
+
end
|
|
61
|
+
snapshot.overlays_for(family, "flag:mixed_high_risk").each do |(_, layer)|
|
|
62
|
+
context << :mixed_high_risk if layer.cover?(ip_int)
|
|
63
|
+
end
|
|
64
|
+
|
|
65
|
+
verdict, provider, sources = decide(snapshot, layers, family, ip_int, flags, category, role, asn)
|
|
66
|
+
|
|
67
|
+
Result.new(
|
|
68
|
+
ip: ip_string, verdict: verdict, asn: asn,
|
|
69
|
+
as_org: snapshot.org_name(asn), category: category, network_role: role,
|
|
70
|
+
provider: provider, sources: sources, flags: flags,
|
|
71
|
+
context_flags: context, unrouted: asn.nil?
|
|
72
|
+
)
|
|
73
|
+
end
|
|
74
|
+
|
|
75
|
+
def decide(snapshot, layers, family, ip_int, flags, category, role, asn) # rubocop:disable Metrics
|
|
76
|
+
# 3 — relay
|
|
77
|
+
if (hit = overlay_hit(snapshot, family, "relay", ip_int))
|
|
78
|
+
return [:relay, hit.provider, [hit.id.to_sym]]
|
|
79
|
+
end
|
|
80
|
+
|
|
81
|
+
# 4 — tor
|
|
82
|
+
if (hit = overlay_hit(snapshot, family, "tor_exit", ip_int))
|
|
83
|
+
return [:tor_exit, hit.provider, [hit.id.to_sym]]
|
|
84
|
+
end
|
|
85
|
+
|
|
86
|
+
# 5 — vpn ranges. Provider-attributed Tier B lists are consulted
|
|
87
|
+
# BEFORE the anonymous canonical overlay on purpose: when both match
|
|
88
|
+
# (common — X4B covers most VPN hosting space), the verdict is
|
|
89
|
+
# identical but "ProtonVPN" beats provider=nil for explainability.
|
|
90
|
+
if (hit = overlay_hit(snapshot, family, "vpn", ip_int))
|
|
91
|
+
return [:vpn, hit.provider, [hit.id.to_sym]]
|
|
92
|
+
end
|
|
93
|
+
|
|
94
|
+
return [:vpn, nil, [:x4b_vpn]] if layers.vpn.cover?(ip_int)
|
|
95
|
+
|
|
96
|
+
# 6 — vpn by ASN flag
|
|
97
|
+
return [:vpn, nil, [:asn_vpn_provider]] if flags.anybits?(BinaryFormat::FLAG_VPN_PROVIDER)
|
|
98
|
+
|
|
99
|
+
# 7 — enterprise gateway (flag, then Tier B ranges)
|
|
100
|
+
return [:enterprise_gateway, nil, [:asn_enterprise_gw]] if flags.anybits?(BinaryFormat::FLAG_ENTERPRISE_GW)
|
|
101
|
+
|
|
102
|
+
if (hit = overlay_hit(snapshot, family, "enterprise_gateway", ip_int))
|
|
103
|
+
return [:enterprise_gateway, hit.provider, [hit.id.to_sym]]
|
|
104
|
+
end
|
|
105
|
+
|
|
106
|
+
# 8 — cloud provider ranges (provider attribution is the value-add)
|
|
107
|
+
if (hit = overlay_hit(snapshot, family, "hosting", ip_int))
|
|
108
|
+
return [:hosting, hit.provider, [hit.id.to_sym]]
|
|
109
|
+
end
|
|
110
|
+
|
|
111
|
+
# 9 — canonical datacenter overlay
|
|
112
|
+
return [:hosting, nil, [:x4b_dc]] if layers.dc.cover?(ip_int)
|
|
113
|
+
|
|
114
|
+
# 10 — hosting by ASN signal
|
|
115
|
+
if flags.anybits?(BinaryFormat::FLAG_BAD_ASN | BinaryFormat::FLAG_HOSTING_EXTRA | BinaryFormat::FLAG_CDN) ||
|
|
116
|
+
category == "hosting"
|
|
117
|
+
return [:hosting, nil, hosting_sources(flags, category)]
|
|
118
|
+
end
|
|
119
|
+
|
|
120
|
+
# 11 — mobile
|
|
121
|
+
return [:mobile, nil, [:asn_mobile_carrier]] if flags.anybits?(BinaryFormat::FLAG_MOBILE)
|
|
122
|
+
|
|
123
|
+
# 12–17 — category ladder
|
|
124
|
+
if category == "isp"
|
|
125
|
+
return role == "tier1_transit" ? [:unknown, nil, [:isp_transit_ambiguous]] : [:residential_isp, nil, [:asn_category]]
|
|
126
|
+
end
|
|
127
|
+
return [:business, nil, [:asn_category]] if category == "business"
|
|
128
|
+
return [:education, nil, [:asn_category]] if category == "education_research"
|
|
129
|
+
return [:government, nil, [:asn_category]] if category == "government_admin"
|
|
130
|
+
return [:unknown, nil, [:asn_no_category]] if asn
|
|
131
|
+
|
|
132
|
+
# 18 — unrouted
|
|
133
|
+
[:unknown, nil, [:unrouted]]
|
|
134
|
+
end
|
|
135
|
+
|
|
136
|
+
def overlay_hit(snapshot, family, maps_to, ip_int)
|
|
137
|
+
snapshot.overlays_for(family, maps_to).each do |(entry, layer)|
|
|
138
|
+
return entry if layer.cover?(ip_int)
|
|
139
|
+
end
|
|
140
|
+
nil
|
|
141
|
+
end
|
|
142
|
+
|
|
143
|
+
def hosting_sources(flags, category)
|
|
144
|
+
sources = []
|
|
145
|
+
sources << :asn_bad_asn if flags.anybits?(BinaryFormat::FLAG_BAD_ASN)
|
|
146
|
+
sources << :asn_hosting_extra if flags.anybits?(BinaryFormat::FLAG_HOSTING_EXTRA)
|
|
147
|
+
sources << :asn_cdn if flags.anybits?(BinaryFormat::FLAG_CDN)
|
|
148
|
+
sources << :asn_category if category == "hosting"
|
|
149
|
+
sources
|
|
150
|
+
end
|
|
151
|
+
end
|
|
152
|
+
end
|
|
@@ -0,0 +1,163 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "logger"
|
|
4
|
+
require "tmpdir"
|
|
5
|
+
|
|
6
|
+
module OpenASN
|
|
7
|
+
class Configuration
|
|
8
|
+
# Where downloaded artifacts + Tier B overlays live. Default:
|
|
9
|
+
# Rails.root/storage/openasn (survives deploys on most setups, and
|
|
10
|
+
# storage/ is gitignored by Rails convention), else Dir.tmpdir/openasn.
|
|
11
|
+
# Heads-up for containerized deploys: put this on a persistent volume
|
|
12
|
+
# or every boot starts from the bundled seed + a fresh download.
|
|
13
|
+
attr_writer :data_dir
|
|
14
|
+
|
|
15
|
+
# Measured on the real dataset, full classify (parse + all layers):
|
|
16
|
+
# :packed → ~11MB data resident; ~15µs/lookup on Apple Silicon,
|
|
17
|
+
# ~24µs on GitHub's shared CI runners (default; right for web apps)
|
|
18
|
+
# :arrays → several× the memory; ~9µs/lookup (~1.6x faster — the raw
|
|
19
|
+
# range probe is ~2µs, but IP parsing + overlay checks
|
|
20
|
+
# dominate, so the full-lookup win is smaller than that
|
|
21
|
+
# suggests). For lookup-heavy batch pipelines.
|
|
22
|
+
attr_reader :memory_mode
|
|
23
|
+
|
|
24
|
+
# Whether UpdateJob/boot staleness checks may refresh data automatically.
|
|
25
|
+
attr_accessor :auto_update
|
|
26
|
+
|
|
27
|
+
# Rolling-release base URL. Override to self-host artifacts (air-gapped
|
|
28
|
+
# deploys, internal mirrors). Must end with "/".
|
|
29
|
+
attr_writer :release_url
|
|
30
|
+
|
|
31
|
+
# Per-source Tier B switches (see fetch-manifest.json in the data repo).
|
|
32
|
+
# Keys are FEATURE names, not source ids — they fan out:
|
|
33
|
+
# apple_relay → apple_private_relay (:relay)
|
|
34
|
+
# tor → tor_exits (:tor_exit)
|
|
35
|
+
# clouds → aws gcp azure oracle digitalocean linode vultr
|
|
36
|
+
# + cloudflare_ranges context flag
|
|
37
|
+
# vpn_providers → protonvpn mullvad ivpn pia airvpn windscribe
|
|
38
|
+
# privado riseup wlvpn worldvpn ovpn
|
|
39
|
+
# anonine
|
|
40
|
+
# (exact provider-attributed VPN exit/server IPs)
|
|
41
|
+
# vpn_heavy → nordvpn (large/fragile provider API)
|
|
42
|
+
# vpn_dns → surfshark ipvanish privatevpn purevpn torguard fastestvpn vpnsecure
|
|
43
|
+
# tunnelbear strongvpn vyprvpn giganews slickvpn
|
|
44
|
+
# azirevpn vpn.ac trust.zone
|
|
45
|
+
# (provider hostnames resolved locally; opt-in)
|
|
46
|
+
# public_relays → vpngate vpnbook freevpn.us (volunteer/free public VPN relays)
|
|
47
|
+
# zscaler → zscaler (:enterprise_gateway ranges)
|
|
48
|
+
# nazgul_mixed → nazgul_mixed (flag only, never :vpn)
|
|
49
|
+
attr_accessor :tier_b
|
|
50
|
+
|
|
51
|
+
# Pin data to a dated release tag (e.g. "v2026.07.05") instead of the
|
|
52
|
+
# rolling latest. For reproducible environments and gradual rollouts.
|
|
53
|
+
attr_accessor :pin_version
|
|
54
|
+
|
|
55
|
+
# Reserved for the post-MVP edge companion (a CDN worker that stamps a
|
|
56
|
+
# trusted classification header). No behavior in this version.
|
|
57
|
+
attr_accessor :trusted_header
|
|
58
|
+
|
|
59
|
+
attr_writer :logger
|
|
60
|
+
|
|
61
|
+
# Reserved for future OpenASN Pro editions. Deliberately inert in the
|
|
62
|
+
# open gem — configuring it does nothing today and never will for the
|
|
63
|
+
# free dataset (see the data repo's open-data contract).
|
|
64
|
+
#
|
|
65
|
+
# Tier C BYOD adapters (bring-your-own MaxMind/IP2Location databases)
|
|
66
|
+
# are post-MVP and deliberately ship NO placeholder keys here: new
|
|
67
|
+
# config keys are additive and non-breaking to introduce later, whereas
|
|
68
|
+
# a placeholder whose shape turns out wrong would force a breaking
|
|
69
|
+
# rename. They'll appear alongside the adapters themselves.
|
|
70
|
+
attr_accessor :api_key
|
|
71
|
+
|
|
72
|
+
TIER_B_DEFAULTS = {
|
|
73
|
+
apple_relay: true,
|
|
74
|
+
tor: true,
|
|
75
|
+
clouds: true,
|
|
76
|
+
zscaler: false, # ASN-level enterprise_gateway overrides already cover Zscaler
|
|
77
|
+
vpn_providers: true,
|
|
78
|
+
vpn_heavy: false, # e.g. NordVPN's ~35MB API response; opt in deliberately
|
|
79
|
+
vpn_dns: false, # provider-published hostnames resolved by local DNS; opt in deliberately
|
|
80
|
+
public_relays: false, # volunteer relays like VPN Gate; useful, but high-churn
|
|
81
|
+
nazgul_mixed: false # semantics broader than VPN; opt-in only
|
|
82
|
+
}.freeze
|
|
83
|
+
|
|
84
|
+
# Feature switch → fetch-manifest source ids.
|
|
85
|
+
TIER_B_SOURCE_MAP = {
|
|
86
|
+
apple_relay: %w[apple_private_relay],
|
|
87
|
+
tor: %w[tor_exits],
|
|
88
|
+
clouds: %w[aws gcp azure oracle digitalocean linode vultr cloudflare_ranges],
|
|
89
|
+
zscaler: %w[zscaler],
|
|
90
|
+
vpn_providers: %w[protonvpn mullvad_relays ivpn_servers pia_servers airvpn_status windscribe_servers
|
|
91
|
+
privadovpn riseup_vpn wlvpn_server_list worldvpn_servers ovpn_status_servers
|
|
92
|
+
anonine_status],
|
|
93
|
+
vpn_heavy: %w[nordvpn_servers],
|
|
94
|
+
vpn_dns: %w[surfshark_generic surfshark_static surfshark_obfuscated ipvanish_openvpn
|
|
95
|
+
privatevpn_openvpn purevpn_openvpn torguard_openvpn_tcp torguard_openvpn_udp
|
|
96
|
+
fastestvpn_tcp fastestvpn_udp vpnsecure_locations tunnelbear_openvpn strongvpn_locations
|
|
97
|
+
vyprvpn_openvpn giganews_vyprvpn_hosts slickvpn_locations azirevpn_locations
|
|
98
|
+
vpnac_status trustzone_servers],
|
|
99
|
+
public_relays: %w[vpngate vpnbook_openvpn freevpn_us_servers],
|
|
100
|
+
nazgul_mixed: %w[nazgul_mixed]
|
|
101
|
+
}.freeze
|
|
102
|
+
|
|
103
|
+
def initialize
|
|
104
|
+
@data_dir = nil
|
|
105
|
+
@memory_mode = :packed
|
|
106
|
+
@auto_update = true
|
|
107
|
+
@release_url = nil
|
|
108
|
+
@tier_b = TIER_B_DEFAULTS.dup
|
|
109
|
+
@pin_version = nil
|
|
110
|
+
@trusted_header = nil
|
|
111
|
+
@logger = nil
|
|
112
|
+
@api_key = nil
|
|
113
|
+
end
|
|
114
|
+
|
|
115
|
+
def data_dir
|
|
116
|
+
@data_dir ||= if defined?(Rails) && Rails.respond_to?(:root) && Rails.root
|
|
117
|
+
File.join(Rails.root.to_s, "storage", "openasn")
|
|
118
|
+
else
|
|
119
|
+
File.join(Dir.tmpdir, "openasn")
|
|
120
|
+
end
|
|
121
|
+
end
|
|
122
|
+
|
|
123
|
+
def memory_mode=(mode)
|
|
124
|
+
raise ArgumentError, "memory_mode must be :packed or :arrays, got #{mode.inspect}" unless %i[packed arrays].include?(mode)
|
|
125
|
+
|
|
126
|
+
@memory_mode = mode
|
|
127
|
+
end
|
|
128
|
+
|
|
129
|
+
def release_url
|
|
130
|
+
return @release_url if @release_url
|
|
131
|
+
return "https://github.com/openasn/openasn/releases/download/#{pin_version}/" if pin_version
|
|
132
|
+
|
|
133
|
+
# Tag-addressed form: the rolling release's TAG is literally "latest",
|
|
134
|
+
# so this path stays pinned to it no matter which release holds
|
|
135
|
+
# GitHub's "Latest" BADGE. The superficially equivalent
|
|
136
|
+
# `releases/latest/download/...` resolves via that badge (whatever
|
|
137
|
+
# release was created last - REST `make_latest` defaults to "true"),
|
|
138
|
+
# and the first weekly dated snapshot stole it once (2026-07-05):
|
|
139
|
+
# default-config clients would have silently fetched up-to-6-day-old
|
|
140
|
+
# data until the next snapshot. Do not "simplify" this back.
|
|
141
|
+
# Refs: https://docs.github.com/en/repositories/releasing-projects-on-github/linking-to-releases
|
|
142
|
+
# https://docs.github.com/en/rest/releases/releases#create-a-release
|
|
143
|
+
# data repo DECISIONS.md D-REL-1
|
|
144
|
+
"https://github.com/openasn/openasn/releases/download/latest/"
|
|
145
|
+
end
|
|
146
|
+
|
|
147
|
+
def logger
|
|
148
|
+
@logger ||= if defined?(Rails) && Rails.respond_to?(:logger) && Rails.logger
|
|
149
|
+
Rails.logger
|
|
150
|
+
else
|
|
151
|
+
Logger.new($stdout, level: Logger::INFO, progname: "openasn")
|
|
152
|
+
end
|
|
153
|
+
end
|
|
154
|
+
|
|
155
|
+
def enabled_tier_b_source_ids
|
|
156
|
+
tier_b.flat_map { |feature, on| on ? TIER_B_SOURCE_MAP.fetch(feature, []) : [] }
|
|
157
|
+
end
|
|
158
|
+
|
|
159
|
+
def user_agent
|
|
160
|
+
"openasn-ruby/#{VERSION} (+https://github.com/openasn/openasn)"
|
|
161
|
+
end
|
|
162
|
+
end
|
|
163
|
+
end
|