openasn 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.simplecov +25 -0
- data/AGENTS.md +23 -0
- data/CHANGELOG.md +52 -0
- data/LICENSE.txt +21 -0
- data/README.md +242 -0
- data/Rakefile +34 -0
- data/lib/generators/openasn/install/install_generator.rb +68 -0
- data/lib/generators/openasn/install/templates/openasn.rb +68 -0
- data/lib/openasn/binary_format.rb +305 -0
- data/lib/openasn/cidr_utils.rb +49 -0
- data/lib/openasn/classifier.rb +152 -0
- data/lib/openasn/configuration.rb +163 -0
- data/lib/openasn/data/seed/fetch-manifest.json +589 -0
- data/lib/openasn/data/seed/manifest.json +99 -0
- data/lib/openasn/data/seed/openasn-ipv4.bin +0 -0
- data/lib/openasn/data/seed/openasn-ipv6.bin +0 -0
- data/lib/openasn/dataset.rb +117 -0
- data/lib/openasn/errors.rb +22 -0
- data/lib/openasn/http_client.rb +86 -0
- data/lib/openasn/ip.rb +70 -0
- data/lib/openasn/middleware.rb +49 -0
- data/lib/openasn/overlay_store.rb +124 -0
- data/lib/openasn/parsers.rb +515 -0
- data/lib/openasn/railtie.rb +30 -0
- data/lib/openasn/result.rb +93 -0
- data/lib/openasn/snapshot.rb +171 -0
- data/lib/openasn/special_ranges.rb +44 -0
- data/lib/openasn/tier_b.rb +223 -0
- data/lib/openasn/update_job.rb +32 -0
- data/lib/openasn/updater.rb +173 -0
- data/lib/openasn/version.rb +5 -0
- data/lib/openasn.rb +115 -0
- metadata +85 -0
|
@@ -0,0 +1,171 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "digest"
|
|
4
|
+
require "json"
|
|
5
|
+
|
|
6
|
+
module OpenASN
|
|
7
|
+
# An immutable, fully-loaded dataset: canonical artifacts (both address
|
|
8
|
+
# families) + org names + Tier B overlays, plus the metadata to explain
|
|
9
|
+
# where it all came from. Snapshots are built off the hot path and
|
|
10
|
+
# swapped in with a single ivar assignment (see Dataset) — readers never
|
|
11
|
+
# see partial state, and there are no locks anywhere near lookups.
|
|
12
|
+
class Snapshot
|
|
13
|
+
SEED_DIR = File.expand_path("data/seed", __dir__)
|
|
14
|
+
|
|
15
|
+
Family = Struct.new(:base, :vpn, :dc, :relay, keyword_init: true)
|
|
16
|
+
|
|
17
|
+
attr_reader :v4, :v6, :orgs, :overlays, :build_id, :build_ts, :loaded_at,
|
|
18
|
+
:origin, :tier_b_status, :record_counts
|
|
19
|
+
|
|
20
|
+
# Build from the best available data:
|
|
21
|
+
# 1. data_dir artifacts (downloaded by the updater) when present+valid
|
|
22
|
+
# 2. the gem's bundled seed otherwise (first boot, corrupted dir, …)
|
|
23
|
+
# A corrupt downloaded artifact NEVER crashes boot — parsing happens
|
|
24
|
+
# INSIDE the fallback boundary (a truncated download that still reads
|
|
25
|
+
# fine must not take the app down); we log and fall back to the seed,
|
|
26
|
+
# and the next update cycle re-downloads. If the bundled seed itself
|
|
27
|
+
# fails to parse, the gem is broken and raising IS correct.
|
|
28
|
+
def self.build(config)
|
|
29
|
+
data_dir = config.data_dir
|
|
30
|
+
mode = config.memory_mode
|
|
31
|
+
|
|
32
|
+
origin = :data_dir
|
|
33
|
+
begin
|
|
34
|
+
v4_bytes, v6_bytes, manifest = read_artifacts(data_dir)
|
|
35
|
+
raise Errno::ENOENT, "no artifacts in data_dir" unless v4_bytes
|
|
36
|
+
|
|
37
|
+
v4_parsed, v6_parsed = parse_pair(v4_bytes, v6_bytes, mode)
|
|
38
|
+
rescue StandardError => e
|
|
39
|
+
unless e.is_a?(Errno::ENOENT)
|
|
40
|
+
config.logger.warn("openasn: data_dir artifacts unusable (#{e.message}); falling back to bundled seed")
|
|
41
|
+
end
|
|
42
|
+
origin = :seed
|
|
43
|
+
v4_bytes = File.binread(File.join(SEED_DIR, "openasn-ipv4.bin"))
|
|
44
|
+
v6_bytes = File.binread(File.join(SEED_DIR, "openasn-ipv6.bin"))
|
|
45
|
+
manifest = JSON.parse(File.read(File.join(SEED_DIR, "manifest.json")))
|
|
46
|
+
v4_parsed, v6_parsed = parse_pair(v4_bytes, v6_bytes, mode)
|
|
47
|
+
end
|
|
48
|
+
|
|
49
|
+
orgs = load_orgs(data_dir, config)
|
|
50
|
+
store = OverlayStore.new(data_dir)
|
|
51
|
+
overlays = store.load(config.enabled_tier_b_source_ids, mode)
|
|
52
|
+
|
|
53
|
+
new(v4_parsed, v6_parsed, orgs, overlays, manifest, origin, store)
|
|
54
|
+
end
|
|
55
|
+
|
|
56
|
+
def self.parse_pair(v4_bytes, v6_bytes, mode)
|
|
57
|
+
v4_parsed = BinaryFormat.parse_artifact(v4_bytes, mode: mode)
|
|
58
|
+
v6_parsed = BinaryFormat.parse_artifact(v6_bytes, mode: mode)
|
|
59
|
+
raise FormatError, "openasn-ipv4.bin is not an IPv4 artifact" unless v4_parsed[:family] == :ipv4
|
|
60
|
+
raise FormatError, "openasn-ipv6.bin is not an IPv6 artifact" unless v6_parsed[:family] == :ipv6
|
|
61
|
+
|
|
62
|
+
[v4_parsed, v6_parsed]
|
|
63
|
+
end
|
|
64
|
+
|
|
65
|
+
def self.read_artifacts(data_dir)
|
|
66
|
+
v4 = File.join(data_dir, "openasn-ipv4.bin")
|
|
67
|
+
v6 = File.join(data_dir, "openasn-ipv6.bin")
|
|
68
|
+
mf = File.join(data_dir, "manifest.json")
|
|
69
|
+
return nil unless File.exist?(v4) && File.exist?(v6)
|
|
70
|
+
|
|
71
|
+
manifest = File.exist?(mf) ? JSON.parse(File.read(mf)) : {}
|
|
72
|
+
verify_manifest_hashes!(manifest, "openasn-ipv4.bin" => v4, "openasn-ipv6.bin" => v6)
|
|
73
|
+
[File.binread(v4), File.binread(v6), manifest]
|
|
74
|
+
end
|
|
75
|
+
|
|
76
|
+
def self.verify_manifest_hashes!(manifest, files)
|
|
77
|
+
by_name = (manifest["files"] || []).to_h { |f| [f["name"], f] }
|
|
78
|
+
return if by_name.empty? # legacy/manual installs without checksums still parse through FORMAT.md.
|
|
79
|
+
|
|
80
|
+
files.each do |name, path|
|
|
81
|
+
expected = by_name.dig(name, "sha256")
|
|
82
|
+
raise IntegrityError, "manifest missing checksum for #{name}" if expected.to_s.empty?
|
|
83
|
+
|
|
84
|
+
actual = Digest::SHA256.file(path).hexdigest
|
|
85
|
+
next if actual == expected
|
|
86
|
+
|
|
87
|
+
# Updater writes manifest.json last as the commit marker. If a
|
|
88
|
+
# process dies during the final rename loop, a fresh boot can see
|
|
89
|
+
# new artifact bytes next to the old manifest. Reject that mixed
|
|
90
|
+
# set here and fall back to the bundled seed instead of serving a
|
|
91
|
+
# cross-build IPv4/IPv6 pair.
|
|
92
|
+
raise IntegrityError, "#{name} checksum does not match manifest"
|
|
93
|
+
end
|
|
94
|
+
end
|
|
95
|
+
|
|
96
|
+
def self.load_orgs(data_dir, config)
|
|
97
|
+
path = File.join(data_dir, "openasn-orgs.bin")
|
|
98
|
+
return nil unless File.exist?(path)
|
|
99
|
+
|
|
100
|
+
BinaryFormat::OrgIndex.load(path)
|
|
101
|
+
rescue StandardError => e
|
|
102
|
+
config.logger.warn("openasn: openasn-orgs.bin unusable (#{e.message}); as_org will be nil until next update")
|
|
103
|
+
nil
|
|
104
|
+
end
|
|
105
|
+
|
|
106
|
+
def initialize(v4_parsed, v6_parsed, orgs, overlays, manifest, origin, store)
|
|
107
|
+
@v4 = Family.new(base: v4_parsed[:base], vpn: v4_parsed[:vpn], dc: v4_parsed[:dc], relay: v4_parsed[:relay])
|
|
108
|
+
@v6 = Family.new(base: v6_parsed[:base], vpn: v6_parsed[:vpn], dc: v6_parsed[:dc], relay: v6_parsed[:relay])
|
|
109
|
+
@orgs = orgs
|
|
110
|
+
@overlays = overlays.freeze
|
|
111
|
+
@build_id = manifest["build_id"]
|
|
112
|
+
@build_ts = v4_parsed[:build_ts]
|
|
113
|
+
@loaded_at = Time.now.utc
|
|
114
|
+
@origin = origin
|
|
115
|
+
@record_counts = {
|
|
116
|
+
base_ipv4: @v4.base.count, vpn_ipv4: @v4.vpn.count, dc_ipv4: @v4.dc.count,
|
|
117
|
+
base_ipv6: @v6.base.count
|
|
118
|
+
}.freeze
|
|
119
|
+
@tier_b_status = build_tier_b_status(store)
|
|
120
|
+
# Precomputed (family, maps_to) -> [[entry, layer], ...] index. The old
|
|
121
|
+
# per-lookup filter_map was fine at 8 overlays but allocation-heavy at
|
|
122
|
+
# 18+ (measured: 42us/lookup vs 15us overlay-less with 11 overlays -
|
|
123
|
+
# most of the gap was these throwaway arrays, not the binary searches).
|
|
124
|
+
# The snapshot is immutable, so build the index exactly once.
|
|
125
|
+
@overlay_index = {}
|
|
126
|
+
@overlays.each do |o|
|
|
127
|
+
{ ipv4: o.v4, ipv6: o.v6 }.each do |fam, layer|
|
|
128
|
+
next unless layer
|
|
129
|
+
|
|
130
|
+
(@overlay_index[[fam, o.maps_to]] ||= []) << [o, layer].freeze
|
|
131
|
+
end
|
|
132
|
+
end
|
|
133
|
+
@overlay_index.each_value(&:freeze)
|
|
134
|
+
@overlay_index.freeze
|
|
135
|
+
freeze
|
|
136
|
+
end
|
|
137
|
+
|
|
138
|
+
def family(fam) = fam == :ipv4 ? @v4 : @v6
|
|
139
|
+
|
|
140
|
+
EMPTY_OVERLAYS = [].freeze
|
|
141
|
+
|
|
142
|
+
# Overlays for one family in a stable order (executor wrote them; order
|
|
143
|
+
# among same-precedence overlays doesn't affect verdicts, only which
|
|
144
|
+
# provider gets attribution on exotic multi-overlay hits).
|
|
145
|
+
def overlays_for(fam, maps_to)
|
|
146
|
+
@overlay_index.fetch([fam, maps_to], EMPTY_OVERLAYS)
|
|
147
|
+
end
|
|
148
|
+
|
|
149
|
+
def org_name(asn)
|
|
150
|
+
asn && @orgs ? @orgs.name(asn) : nil
|
|
151
|
+
end
|
|
152
|
+
|
|
153
|
+
def age_seconds
|
|
154
|
+
Time.now.to_i - @build_ts
|
|
155
|
+
end
|
|
156
|
+
|
|
157
|
+
private
|
|
158
|
+
|
|
159
|
+
def build_tier_b_status(store)
|
|
160
|
+
state = store.state["sources"]
|
|
161
|
+
@overlays.to_h do |o|
|
|
162
|
+
s = state[o.id] || {}
|
|
163
|
+
[o.id.to_sym, {
|
|
164
|
+
maps_to: o.maps_to, fetched_at: s["fetched_at"],
|
|
165
|
+
records: { ipv4: s["records_ipv4"], ipv6: s["records_ipv6"] },
|
|
166
|
+
last_error: s["last_error"]
|
|
167
|
+
}]
|
|
168
|
+
end.freeze
|
|
169
|
+
end
|
|
170
|
+
end
|
|
171
|
+
end
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "ipaddr"
|
|
4
|
+
|
|
5
|
+
module OpenASN
|
|
6
|
+
# Hardcoded special-purpose ranges, checked before any data layer.
|
|
7
|
+
# These are IANA facts, not data: they never change with dataset updates.
|
|
8
|
+
module SpecialRanges
|
|
9
|
+
def self.table(cidrs)
|
|
10
|
+
cidrs.map do |(cidr, verdict, rule)|
|
|
11
|
+
r = IPAddr.new(cidr).to_range
|
|
12
|
+
[r.first.to_i, r.last.to_i, verdict, rule].freeze
|
|
13
|
+
end.freeze
|
|
14
|
+
end
|
|
15
|
+
|
|
16
|
+
V4 = table([
|
|
17
|
+
["0.0.0.0/8", :private, :special_reserved],
|
|
18
|
+
["10.0.0.0/8", :private, :special_rfc1918],
|
|
19
|
+
["100.64.0.0/10", :cgnat, :special_cgnat], # RFC 6598 shared address space
|
|
20
|
+
["127.0.0.0/8", :private, :special_loopback],
|
|
21
|
+
["169.254.0.0/16", :private, :special_link_local],
|
|
22
|
+
["172.16.0.0/12", :private, :special_rfc1918],
|
|
23
|
+
["192.168.0.0/16", :private, :special_rfc1918],
|
|
24
|
+
["224.0.0.0/4", :private, :special_multicast],
|
|
25
|
+
["240.0.0.0/4", :private, :special_reserved]
|
|
26
|
+
]).freeze
|
|
27
|
+
|
|
28
|
+
V6 = table([
|
|
29
|
+
["::1/128", :private, :special_loopback],
|
|
30
|
+
["fc00::/7", :private, :special_ula],
|
|
31
|
+
["fe80::/10", :private, :special_link_local]
|
|
32
|
+
]).freeze
|
|
33
|
+
|
|
34
|
+
# -> [verdict, rule] | nil. Linear scan is optimal here: 9 rows max,
|
|
35
|
+
# sorted so RFC1918/loopback hits early; measured faster than any
|
|
36
|
+
# cleverer structure at this size.
|
|
37
|
+
def self.match(ip_int, family)
|
|
38
|
+
(family == :ipv4 ? V4 : V6).each do |(s, e, verdict, rule)|
|
|
39
|
+
return [verdict, rule] if ip_int >= s && ip_int <= e
|
|
40
|
+
end
|
|
41
|
+
nil
|
|
42
|
+
end
|
|
43
|
+
end
|
|
44
|
+
end
|
|
@@ -0,0 +1,223 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "json"
|
|
4
|
+
require "socket"
|
|
5
|
+
require "thread"
|
|
6
|
+
require "timeout"
|
|
7
|
+
|
|
8
|
+
module OpenASN
|
|
9
|
+
# Executes fetch-manifest.json: pulls each enabled Tier B source from its
|
|
10
|
+
# ORIGINAL authority (Apple, the Tor Project, AWS…), parses, merges, and
|
|
11
|
+
# packs it into the local overlay store.
|
|
12
|
+
#
|
|
13
|
+
# Prime directives (each encodes a production lesson):
|
|
14
|
+
# * Per-source isolation: one source failing NEVER touches the others
|
|
15
|
+
# and NEVER raises out of the executor. Failures keep last-good data
|
|
16
|
+
# ("keep_stale") and are recorded in state.json — visible via
|
|
17
|
+
# OpenASN.dataset_info[:tier_b_status].
|
|
18
|
+
# * Honor cadence_hours: a 12h source isn't refetched on every deploy's
|
|
19
|
+
# update run. `force: true` overrides (manual OpenASN.update!(force:)).
|
|
20
|
+
# * Unknown source ids / parser ids are skipped with a warning — old
|
|
21
|
+
# gem versions must survive fetch-manifest evolution.
|
|
22
|
+
# * Every request carries the descriptive User-Agent. These are mostly
|
|
23
|
+
# free/volunteer endpoints; being a good citizen is part of the deal.
|
|
24
|
+
class TierB
|
|
25
|
+
DEFAULT_DNS_THREADS = 16
|
|
26
|
+
MAX_DNS_THREADS = 32
|
|
27
|
+
DNS_TIMEOUT_SECONDS = 4
|
|
28
|
+
|
|
29
|
+
class << self
|
|
30
|
+
attr_accessor :dns_resolver
|
|
31
|
+
end
|
|
32
|
+
self.dns_resolver = lambda do |hostname|
|
|
33
|
+
Socket.getaddrinfo(hostname, nil, Socket::AF_UNSPEC, Socket::SOCK_STREAM).map { |entry| entry[3] }.uniq
|
|
34
|
+
end
|
|
35
|
+
|
|
36
|
+
def initialize(config, http)
|
|
37
|
+
@config = config
|
|
38
|
+
@http = http
|
|
39
|
+
@logger = config.logger
|
|
40
|
+
@store = OverlayStore.new(config.data_dir)
|
|
41
|
+
end
|
|
42
|
+
|
|
43
|
+
# -> true when any overlay changed (snapshot reload needed)
|
|
44
|
+
def execute(force: false)
|
|
45
|
+
manifest = load_manifest
|
|
46
|
+
return false unless manifest
|
|
47
|
+
|
|
48
|
+
enabled = @config.enabled_tier_b_source_ids
|
|
49
|
+
changed = false
|
|
50
|
+
(manifest["sources"] || []).each do |source|
|
|
51
|
+
id = source["id"]
|
|
52
|
+
next unless enabled.include?(id)
|
|
53
|
+
|
|
54
|
+
unless Parsers.known?(source["parser"])
|
|
55
|
+
@logger.warn("openasn: tier B source #{id} uses unknown parser #{source['parser'].inspect} — " \
|
|
56
|
+
"skipping (update the openasn gem to pick it up)")
|
|
57
|
+
next
|
|
58
|
+
end
|
|
59
|
+
|
|
60
|
+
changed |= refresh_source(source, force: force)
|
|
61
|
+
end
|
|
62
|
+
changed
|
|
63
|
+
end
|
|
64
|
+
|
|
65
|
+
private
|
|
66
|
+
|
|
67
|
+
# Freshest available manifest: data_dir (mirrored on canonical update)
|
|
68
|
+
# -> bundled copy from gem release time.
|
|
69
|
+
def load_manifest
|
|
70
|
+
[File.join(@config.data_dir, "fetch-manifest.json"),
|
|
71
|
+
File.join(Snapshot::SEED_DIR, "fetch-manifest.json")].each do |path|
|
|
72
|
+
next unless File.exist?(path)
|
|
73
|
+
|
|
74
|
+
return JSON.parse(File.read(path))
|
|
75
|
+
rescue JSON::ParserError => e
|
|
76
|
+
@logger.warn("openasn: unreadable fetch-manifest at #{path} (#{e.message})")
|
|
77
|
+
end
|
|
78
|
+
nil
|
|
79
|
+
end
|
|
80
|
+
|
|
81
|
+
def refresh_source(source, force:)
|
|
82
|
+
id = source["id"]
|
|
83
|
+
unless force || due?(id, source["cadence_hours"])
|
|
84
|
+
return false
|
|
85
|
+
end
|
|
86
|
+
|
|
87
|
+
urls = resolve_urls(source)
|
|
88
|
+
if urls.empty?
|
|
89
|
+
@store.record_failure(id, "could not resolve source URL")
|
|
90
|
+
return false
|
|
91
|
+
end
|
|
92
|
+
|
|
93
|
+
etag = urls.length == 1 && !force ? @store.source_state(id)["etag"] : nil
|
|
94
|
+
tokens = []
|
|
95
|
+
new_etag = nil
|
|
96
|
+
not_modified = false
|
|
97
|
+
|
|
98
|
+
urls.each do |url|
|
|
99
|
+
response = fetch_source_url(source, url, etag)
|
|
100
|
+
if response == :not_modified
|
|
101
|
+
not_modified = true
|
|
102
|
+
break
|
|
103
|
+
end
|
|
104
|
+
new_etag = response.etag if urls.length == 1
|
|
105
|
+
tokens.concat(Parsers.parse(source["parser"], response.body))
|
|
106
|
+
end
|
|
107
|
+
tokens = resolve_hostnames(tokens, source)
|
|
108
|
+
|
|
109
|
+
if not_modified
|
|
110
|
+
@store.record_fresh(id)
|
|
111
|
+
return false
|
|
112
|
+
end
|
|
113
|
+
|
|
114
|
+
ranges = CidrUtils.ranges_by_family(tokens)
|
|
115
|
+
total = ranges[:ipv4].length + ranges[:ipv6].length
|
|
116
|
+
if total.zero?
|
|
117
|
+
# An empty security list is far more likely upstream breakage than
|
|
118
|
+
# reality — keep whatever we had (keep_stale), record loudly.
|
|
119
|
+
@store.record_failure(id, "parsed 0 ranges — upstream format changed? keeping stale data")
|
|
120
|
+
return false
|
|
121
|
+
end
|
|
122
|
+
|
|
123
|
+
@store.write(id, maps_to: source["maps_to"], provider: source["provider"],
|
|
124
|
+
etag: new_etag, ranges_by_family: ranges)
|
|
125
|
+
@logger.info("openasn: tier B #{id}: #{ranges[:ipv4].length} v4 + #{ranges[:ipv6].length} v6 ranges")
|
|
126
|
+
true
|
|
127
|
+
rescue StandardError => e
|
|
128
|
+
# keep_stale: failure is recorded, previous overlay files stay live.
|
|
129
|
+
@store.record_failure(id, "#{e.class}: #{e.message}")
|
|
130
|
+
@logger.warn("openasn: tier B #{id} failed (#{e.message}); keeping stale data")
|
|
131
|
+
false
|
|
132
|
+
end
|
|
133
|
+
|
|
134
|
+
def fetch_source_url(source, url, etag)
|
|
135
|
+
if source["method"].to_s.upcase == "POST"
|
|
136
|
+
@http.post_form(url, source["form"] || {})
|
|
137
|
+
else
|
|
138
|
+
@http.get(url, etag: etag)
|
|
139
|
+
end
|
|
140
|
+
end
|
|
141
|
+
|
|
142
|
+
def due?(id, cadence_hours)
|
|
143
|
+
last = @store.fetched_at(id)
|
|
144
|
+
return true unless last
|
|
145
|
+
|
|
146
|
+
(Time.now - last) >= (cadence_hours || 24) * 3600 * 0.99 # 1% slack so daily jobs don't skip-drift
|
|
147
|
+
end
|
|
148
|
+
|
|
149
|
+
def resolve_urls(source)
|
|
150
|
+
urls = []
|
|
151
|
+
if source["resolver"] == "azure_download_page"
|
|
152
|
+
url = resolve_azure(source["page_url"])
|
|
153
|
+
urls << url if url
|
|
154
|
+
elsif source["url"]
|
|
155
|
+
urls << source["url"]
|
|
156
|
+
end
|
|
157
|
+
urls.concat(source["urls"]) if source["urls"].is_a?(Array)
|
|
158
|
+
urls << source["url_ipv6"] if source["url_ipv6"]
|
|
159
|
+
urls
|
|
160
|
+
end
|
|
161
|
+
|
|
162
|
+
def resolve_hostnames(tokens, source)
|
|
163
|
+
return tokens unless source["resolve_hostnames"]
|
|
164
|
+
|
|
165
|
+
direct = []
|
|
166
|
+
hosts = []
|
|
167
|
+
tokens.each do |token|
|
|
168
|
+
token = token.to_s.strip
|
|
169
|
+
next if token.empty?
|
|
170
|
+
|
|
171
|
+
if CidrUtils.parse(token)
|
|
172
|
+
direct << token
|
|
173
|
+
elsif hostname?(token)
|
|
174
|
+
hosts << token.downcase
|
|
175
|
+
end
|
|
176
|
+
end
|
|
177
|
+
hosts.uniq!
|
|
178
|
+
return direct if hosts.empty?
|
|
179
|
+
|
|
180
|
+
resolved = resolve_hosts(hosts, source)
|
|
181
|
+
@logger.info("openasn: tier B #{source['id']}: resolved #{resolved.length} IPs from #{hosts.length} hostnames")
|
|
182
|
+
direct + resolved
|
|
183
|
+
end
|
|
184
|
+
|
|
185
|
+
def resolve_hosts(hosts, source)
|
|
186
|
+
threads = [[source["dns_threads"] || DEFAULT_DNS_THREADS, MAX_DNS_THREADS].min, hosts.length].min
|
|
187
|
+
queue = Queue.new
|
|
188
|
+
hosts.each { |host| queue << host }
|
|
189
|
+
resolved = []
|
|
190
|
+
mutex = Mutex.new
|
|
191
|
+
|
|
192
|
+
workers = threads.times.map do
|
|
193
|
+
Thread.new do
|
|
194
|
+
loop do
|
|
195
|
+
host = queue.pop(true)
|
|
196
|
+
ips = Timeout.timeout(DNS_TIMEOUT_SECONDS) { self.class.dns_resolver.call(host) }
|
|
197
|
+
mutex.synchronize { resolved.concat(ips) }
|
|
198
|
+
rescue ThreadError
|
|
199
|
+
break
|
|
200
|
+
rescue StandardError
|
|
201
|
+
next
|
|
202
|
+
end
|
|
203
|
+
end
|
|
204
|
+
end
|
|
205
|
+
workers.each(&:join)
|
|
206
|
+
resolved.uniq
|
|
207
|
+
end
|
|
208
|
+
|
|
209
|
+
def hostname?(token)
|
|
210
|
+
token.match?(/\A(?=.{1,253}\z)(?:[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?\.)+[a-z]{2,63}\z/i)
|
|
211
|
+
end
|
|
212
|
+
|
|
213
|
+
# Azure's actual JSON URL rotates weekly behind the download page.
|
|
214
|
+
# Scrape it; on any failure return nil (-> keep stale, try tomorrow).
|
|
215
|
+
def resolve_azure(page_url)
|
|
216
|
+
html = @http.get(page_url).body
|
|
217
|
+
html[%r{https://download\.microsoft\.com/download/[^"'\s]+ServiceTags_Public_\d+\.json}]
|
|
218
|
+
rescue StandardError => e
|
|
219
|
+
@logger.warn("openasn: azure page resolution failed (#{e.message})")
|
|
220
|
+
nil
|
|
221
|
+
end
|
|
222
|
+
end
|
|
223
|
+
end
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
# Defined only when ActiveJob is available (required from the Railtie, or
|
|
4
|
+
# require it yourself in non-Rails ActiveJob apps AFTER loading ActiveJob).
|
|
5
|
+
#
|
|
6
|
+
# Scheduling is app-side — with Solid Queue (Rails 8 default), in
|
|
7
|
+
# config/recurring.yml:
|
|
8
|
+
#
|
|
9
|
+
# production:
|
|
10
|
+
# openasn_update:
|
|
11
|
+
# class: OpenASN::UpdateJob
|
|
12
|
+
# schedule: every day at 4:12am UTC # off-hour on purpose: be kind
|
|
13
|
+
# # to the volunteer-run upstreams
|
|
14
|
+
#
|
|
15
|
+
# (The install generator wires this for you.) Data freshness NEVER flows
|
|
16
|
+
# through gem releases — this job is the only moving part.
|
|
17
|
+
if defined?(ActiveJob::Base)
|
|
18
|
+
module OpenASN
|
|
19
|
+
class UpdateJob < ActiveJob::Base
|
|
20
|
+
queue_as :default
|
|
21
|
+
|
|
22
|
+
# Transient network trouble self-heals on the next run; a persistent
|
|
23
|
+
# failure surfaces via logs + dataset_info staleness, not via a
|
|
24
|
+
# crashed-job pile-up.
|
|
25
|
+
retry_on OpenASN::UpdateError, wait: 5.minutes, attempts: 3
|
|
26
|
+
|
|
27
|
+
def perform(force: false)
|
|
28
|
+
OpenASN.update!(force: force)
|
|
29
|
+
end
|
|
30
|
+
end
|
|
31
|
+
end
|
|
32
|
+
end
|
|
@@ -0,0 +1,173 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "digest"
|
|
4
|
+
require "fileutils"
|
|
5
|
+
require "json"
|
|
6
|
+
|
|
7
|
+
module OpenASN
|
|
8
|
+
# The data refresh cycle (what OpenASN.update! / UpdateJob runs):
|
|
9
|
+
#
|
|
10
|
+
# 1. Canonical: GET {release_url}manifest.json (ETag-conditional).
|
|
11
|
+
# When it changed, download each artifact, VERIFY ITS SHA-256
|
|
12
|
+
# against the manifest, write to *.tmp, File.rename into place
|
|
13
|
+
# (atomic on POSIX — a crashed update can never leave a torn file),
|
|
14
|
+
# then write manifest.json LAST — it is the commit marker other
|
|
15
|
+
# processes watch for (Dataset#maybe_reload_from_disk).
|
|
16
|
+
# 2. Tier B: execute the fetch-manifest for enabled sources (TierB).
|
|
17
|
+
# Per-source failures keep stale data and never fail the update.
|
|
18
|
+
# 3. Swap the in-process snapshot.
|
|
19
|
+
#
|
|
20
|
+
# Concurrency: a non-blocking flock serializes updaters across processes
|
|
21
|
+
# (first puma worker wins; the rest skip — they'll pick up the files via
|
|
22
|
+
# the freshness probe). Never raises for "someone else is updating".
|
|
23
|
+
class Updater
|
|
24
|
+
CANONICAL_FILES = %w[openasn-ipv4.bin openasn-ipv6.bin openasn-orgs.bin].freeze
|
|
25
|
+
|
|
26
|
+
def initialize(config, dataset)
|
|
27
|
+
@config = config
|
|
28
|
+
@dataset = dataset
|
|
29
|
+
@http = HttpClient.new(user_agent: config.user_agent, logger: config.logger)
|
|
30
|
+
@logger = config.logger
|
|
31
|
+
end
|
|
32
|
+
|
|
33
|
+
# -> :updated | :tier_b_only | :unchanged | :locked
|
|
34
|
+
def run(force: false)
|
|
35
|
+
FileUtils.mkdir_p(@config.data_dir)
|
|
36
|
+
with_lock do
|
|
37
|
+
canonical_changed = refresh_canonical(force: force)
|
|
38
|
+
tier_b_changed = TierB.new(@config, @http).execute(force: force)
|
|
39
|
+
|
|
40
|
+
if canonical_changed || tier_b_changed
|
|
41
|
+
@dataset.reload!
|
|
42
|
+
notify(canonical_changed ? :updated : :tier_b_only)
|
|
43
|
+
canonical_changed ? :updated : :tier_b_only
|
|
44
|
+
else
|
|
45
|
+
:unchanged
|
|
46
|
+
end
|
|
47
|
+
end
|
|
48
|
+
end
|
|
49
|
+
|
|
50
|
+
private
|
|
51
|
+
|
|
52
|
+
def refresh_canonical(force:)
|
|
53
|
+
state = read_state
|
|
54
|
+
begin
|
|
55
|
+
response = @http.get("#{@config.release_url}manifest.json",
|
|
56
|
+
etag: force ? nil : state["manifest_etag"])
|
|
57
|
+
rescue StandardError => e
|
|
58
|
+
# No manifest, no update — but never crash the job: keep-last-good
|
|
59
|
+
# is the contract (the seed or previous download keeps serving).
|
|
60
|
+
raise UpdateError, "could not fetch manifest: #{e.message}" if force
|
|
61
|
+
|
|
62
|
+
@logger.warn("openasn: manifest fetch failed (#{e.message}); keeping current data")
|
|
63
|
+
return false
|
|
64
|
+
end
|
|
65
|
+
return false if response == :not_modified
|
|
66
|
+
|
|
67
|
+
manifest = JSON.parse(response.body)
|
|
68
|
+
if !force && manifest["build_id"] && manifest["build_id"] == current_local_build_id
|
|
69
|
+
# Same build re-served (ETag miss, mirror change…): record the new
|
|
70
|
+
# ETag, skip the downloads.
|
|
71
|
+
write_state(state.merge("manifest_etag" => response.etag))
|
|
72
|
+
return false
|
|
73
|
+
end
|
|
74
|
+
|
|
75
|
+
by_name = (manifest["files"] || []).to_h { |f| [f["name"], f] }
|
|
76
|
+
downloads = CANONICAL_FILES.filter_map do |name|
|
|
77
|
+
meta = by_name[name]
|
|
78
|
+
# Manifest without a required artifact = upstream problem; orgs is
|
|
79
|
+
# optional richness, the .bin artifacts are not.
|
|
80
|
+
if meta.nil?
|
|
81
|
+
raise UpdateError, "release manifest is missing #{name}" unless name == "openasn-orgs.bin"
|
|
82
|
+
|
|
83
|
+
next
|
|
84
|
+
end
|
|
85
|
+
[name, meta]
|
|
86
|
+
end
|
|
87
|
+
|
|
88
|
+
# Two-phase install: download + verify EVERYTHING first, then rename
|
|
89
|
+
# all files in one tight loop. A crash mid-download therefore can't
|
|
90
|
+
# leave a mixed set (new ipv4 + old ipv6) on disk; renames are so
|
|
91
|
+
# close together that the mixed window is effectively gone.
|
|
92
|
+
verified = downloads.map do |(name, meta)|
|
|
93
|
+
body = @http.get("#{@config.release_url}#{name}").body
|
|
94
|
+
actual = Digest::SHA256.hexdigest(body)
|
|
95
|
+
unless actual == meta["sha256"]
|
|
96
|
+
raise IntegrityError,
|
|
97
|
+
"#{name} SHA-256 mismatch (expected #{meta['sha256'][0, 12]}…, got #{actual[0, 12]}…) — " \
|
|
98
|
+
"refusing to install; previous data stays live"
|
|
99
|
+
end
|
|
100
|
+
|
|
101
|
+
tmp = File.join(@config.data_dir, "#{name}.tmp")
|
|
102
|
+
File.binwrite(tmp, body)
|
|
103
|
+
[tmp, File.join(@config.data_dir, name)]
|
|
104
|
+
end
|
|
105
|
+
verified.each { |(tmp, final)| File.rename(tmp, final) }
|
|
106
|
+
|
|
107
|
+
# Also mirror the fetch-manifest so Tier B follows the release's
|
|
108
|
+
# current source list (falls back to the bundled copy when absent).
|
|
109
|
+
begin
|
|
110
|
+
fm = @http.get("#{@config.release_url}fetch-manifest.json").body
|
|
111
|
+
JSON.parse(fm) # only install valid JSON
|
|
112
|
+
tmp = File.join(@config.data_dir, "fetch-manifest.json.tmp")
|
|
113
|
+
File.write(tmp, fm)
|
|
114
|
+
File.rename(tmp, File.join(@config.data_dir, "fetch-manifest.json"))
|
|
115
|
+
rescue StandardError => e
|
|
116
|
+
@logger.warn("openasn: fetch-manifest refresh failed (#{e.message}); using previous/bundled copy")
|
|
117
|
+
end
|
|
118
|
+
|
|
119
|
+
# Commit marker LAST: other processes treat manifest.json's mtime as
|
|
120
|
+
# "new data is fully in place".
|
|
121
|
+
tmp = File.join(@config.data_dir, "manifest.json.tmp")
|
|
122
|
+
File.write(tmp, response.body)
|
|
123
|
+
File.rename(tmp, File.join(@config.data_dir, "manifest.json"))
|
|
124
|
+
|
|
125
|
+
write_state(read_state.merge("manifest_etag" => response.etag,
|
|
126
|
+
"updated_at" => Time.now.utc.iso8601))
|
|
127
|
+
@logger.info("openasn: canonical data updated to build #{manifest['build_id']}")
|
|
128
|
+
true
|
|
129
|
+
end
|
|
130
|
+
|
|
131
|
+
def current_local_build_id
|
|
132
|
+
path = File.join(@config.data_dir, "manifest.json")
|
|
133
|
+
return nil unless File.exist?(path)
|
|
134
|
+
|
|
135
|
+
JSON.parse(File.read(path))["build_id"]
|
|
136
|
+
rescue JSON::ParserError
|
|
137
|
+
nil
|
|
138
|
+
end
|
|
139
|
+
|
|
140
|
+
def read_state
|
|
141
|
+
path = File.join(@config.data_dir, "update-state.json")
|
|
142
|
+
File.exist?(path) ? JSON.parse(File.read(path)) : {}
|
|
143
|
+
rescue JSON::ParserError
|
|
144
|
+
{}
|
|
145
|
+
end
|
|
146
|
+
|
|
147
|
+
def write_state(state)
|
|
148
|
+
path = File.join(@config.data_dir, "update-state.json")
|
|
149
|
+
File.write("#{path}.tmp", JSON.pretty_generate(state))
|
|
150
|
+
File.rename("#{path}.tmp", path)
|
|
151
|
+
end
|
|
152
|
+
|
|
153
|
+
def with_lock
|
|
154
|
+
lock_path = File.join(@config.data_dir, ".update.lock")
|
|
155
|
+
File.open(lock_path, File::RDWR | File::CREAT, 0o644) do |f|
|
|
156
|
+
unless f.flock(File::LOCK_EX | File::LOCK_NB)
|
|
157
|
+
@logger.info("openasn: another process is updating — skipping")
|
|
158
|
+
return :locked
|
|
159
|
+
end
|
|
160
|
+
yield
|
|
161
|
+
end
|
|
162
|
+
end
|
|
163
|
+
|
|
164
|
+
def notify(kind)
|
|
165
|
+
return unless defined?(ActiveSupport::Notifications)
|
|
166
|
+
|
|
167
|
+
ActiveSupport::Notifications.instrument("openasn.updated", kind: kind,
|
|
168
|
+
info: @dataset.info)
|
|
169
|
+
rescue StandardError => e
|
|
170
|
+
@logger.warn("openasn: notification hook failed (#{e.message})")
|
|
171
|
+
end
|
|
172
|
+
end
|
|
173
|
+
end
|