network-infra-utility 0.6.0 → 0.8.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.gitignore +6 -0
- data/CHANGELOG.md +31 -0
- data/GUIDE.md +201 -212
- data/README.md +1 -6
- data/Rakefile +1 -1
- data/bin/gen-get +144 -0
- data/bin/geo-api +40 -1
- data/bin/geo-get +18 -36
- data/bin/ngeo-get +253 -0
- data/bin/packbit +227 -0
- data/bin/probe +134 -0
- data/document/reference/DNS/346/212/200/346/234/257/346/226/207/346/241/243.md +912 -0
- data/network-infra-utility.gemspec +6 -4
- data/network.rb +19 -1
- data/service/geodb/GeoAPI.md +64 -0
- data/service/geodb/api.rb +77 -19
- data/service/geoquery/GeoQuery.md +309 -0
- data/service/geoquery/cache.rb +78 -0
- data/service/geoquery/geo_cache.rb +233 -0
- data/service/geoquery/geoquery.rb +327 -0
- data/service/geoquery/local.rb +151 -0
- data/service/geoquery/merge.rb +52 -0
- data/service/geoquery/normalize.rb +102 -0
- data/service/geoquery/online.rb +136 -0
- data/service/simlab/SimLab.md +140 -0
- data/service/simlab/bin/simctl +152 -0
- data/service/simlab/lib/simlab.rb +124 -0
- data/service/simlab/lib/simlab_engine.rb +146 -0
- data/service/simlab/lib/simlab_observer.rb +176 -0
- data/service/simlab/lib/simlab_router.rb +330 -0
- data/service/simlab/lib/simlab_scenario.rb +200 -0
- data/service/simlab/lib/simlab_switch.rb +182 -0
- data/service/simlab/lib/simlab_topology.rb +196 -0
- data/service/simlab/lib/simlab_traffic.rb +167 -0
- data/support/basic/packet.rb +167 -0
- data/support/routing/bgp.rb +279 -0
- data/support/routing/lpm_trie.rb +145 -0
- data/support/routing/ospf.rb +243 -0
- data/support/routing/prefix.rb +155 -0
- data/support/routing/rib.rb +167 -0
- data/support/routing/rip.rb +223 -0
- data/support/routing/static_route.rb +102 -0
- data/support/switching/frame_forward.rb +87 -0
- data/support/switching/mac_table.rb +127 -0
- data/support/switching/stp.rb +230 -0
- data/support/switching/vlan.rb +155 -0
- data/tool/packbit/README.md +136 -0
- data/tool/packbit/config/five-tuple.yml +25 -0
- data/tool/packbit/config/flow.yml +17 -0
- data/tool/packbit/config/packbit.yml +41 -0
- data/tool/packbit/erl/packbit.erl +588 -0
- data/tool/probe/README.md +88 -0
- data/tool/probe/lib/probe.rb +54 -0
- data/tool/probe/lib/probe_base.rb +98 -0
- data/tool/probe/lib/probe_dns.rb +64 -0
- data/tool/probe/lib/probe_icmp.rb +71 -0
- data/tool/probe/lib/probe_ntp.rb +51 -0
- data/tool/probe/lib/probe_radius.rb +58 -0
- data/tool/probe/lib/probe_registry.rb +44 -0
- data/tool/probe/lib/probe_result.rb +75 -0
- data/tool/probe/lib/probe_snmp.rb +114 -0
- data/tool/probe/lib/probe_tcp.rb +82 -0
- data/tool/probe/lib/probe_version.rb +7 -0
- data/version.rb +1 -1
- metadata +74 -2
|
@@ -0,0 +1,78 @@
|
|
|
1
|
+
# coding: utf-8
|
|
2
|
+
# frozen_string_literal: true
|
|
3
|
+
|
|
4
|
+
require "json"
|
|
5
|
+
|
|
6
|
+
module GeoQuery
|
|
7
|
+
# ------------------------------------------------------------------
|
|
8
|
+
# JSON 文件缓存 — 移植自 ip_geo_lookup.py 的缓存设计
|
|
9
|
+
# ------------------------------------------------------------------
|
|
10
|
+
# 结构: { "ip" => { result..., "ts" => 1716000000 } }
|
|
11
|
+
# 线程安全 (Mutex); 不可达 (unreachable) 类结果不写入缓存。
|
|
12
|
+
class Cache
|
|
13
|
+
attr_reader :path
|
|
14
|
+
|
|
15
|
+
# 默认缓存根目录: ENV["NGEO_CACHE_DIR"] || ~/.network-infra-utility/
|
|
16
|
+
def self.default_dir
|
|
17
|
+
dir = ENV["NGEO_CACHE_DIR"].to_s
|
|
18
|
+
dir.empty? ? File.join(Dir.home, ".network-infra-utility") : File.expand_path(dir)
|
|
19
|
+
end
|
|
20
|
+
|
|
21
|
+
def initialize(path)
|
|
22
|
+
@path = path
|
|
23
|
+
@lock = Mutex.new
|
|
24
|
+
@data = load
|
|
25
|
+
end
|
|
26
|
+
|
|
27
|
+
# 命中返回结果 Hash (附加 "cached" => true), 未命中返回 nil
|
|
28
|
+
def get(ip)
|
|
29
|
+
@lock.synchronize do
|
|
30
|
+
hit = @data[ip]
|
|
31
|
+
hit ? hit.merge("cached" => true) : nil
|
|
32
|
+
end
|
|
33
|
+
end
|
|
34
|
+
|
|
35
|
+
# 写入一条 (网络类不可达状态不落盘, 下次重查)
|
|
36
|
+
def put(ip, result)
|
|
37
|
+
return if result["state"] == "unreachable"
|
|
38
|
+
@lock.synchronize do
|
|
39
|
+
@data[ip] = result.merge("ts" => Time.now.to_i)
|
|
40
|
+
end
|
|
41
|
+
end
|
|
42
|
+
|
|
43
|
+
def save
|
|
44
|
+
@lock.synchronize do
|
|
45
|
+
dir = File.dirname(@path)
|
|
46
|
+
require "fileutils"
|
|
47
|
+
FileUtils.mkdir_p(dir)
|
|
48
|
+
File.binwrite(@path, JSON.pretty_generate(@data))
|
|
49
|
+
end
|
|
50
|
+
@path
|
|
51
|
+
end
|
|
52
|
+
|
|
53
|
+
def size
|
|
54
|
+
@lock.synchronize { @data.size }
|
|
55
|
+
end
|
|
56
|
+
|
|
57
|
+
def stats
|
|
58
|
+
loaded = @lock.synchronize { @data.dup }
|
|
59
|
+
states = Hash.new(0)
|
|
60
|
+
loaded.each_value { |v| states[v["state"]] += 1 }
|
|
61
|
+
{
|
|
62
|
+
"total" => loaded.size,
|
|
63
|
+
"file" => @path,
|
|
64
|
+
"size_kb" => File.exist?(@path) ? (File.size(@path) / 1024.0).round(1) : 0,
|
|
65
|
+
"states" => states,
|
|
66
|
+
}
|
|
67
|
+
end
|
|
68
|
+
|
|
69
|
+
private
|
|
70
|
+
|
|
71
|
+
def load
|
|
72
|
+
return {} unless File.exist?(@path)
|
|
73
|
+
JSON.parse(File.binread(@path))
|
|
74
|
+
rescue JSON::ParserError, Errno::ENOENT
|
|
75
|
+
{}
|
|
76
|
+
end
|
|
77
|
+
end
|
|
78
|
+
end
|
|
@@ -0,0 +1,233 @@
|
|
|
1
|
+
# coding: utf-8
|
|
2
|
+
# frozen_string_literal: true
|
|
3
|
+
|
|
4
|
+
require "json"
|
|
5
|
+
require_relative "normalize" # GeoQuery.valid_ip? (服务端可单独加载本文件)
|
|
6
|
+
|
|
7
|
+
module GeoQuery
|
|
8
|
+
# ------------------------------------------------------------------
|
|
9
|
+
# GEO_CACHE 外带缓存目录 — geocacheXXXXXXXX.json 文件集的读取/写入/合并
|
|
10
|
+
# ------------------------------------------------------------------
|
|
11
|
+
# 文件命名: geocacheYYYYMMDD.json (XXXXXXXX 为 8 位日期时间标签)
|
|
12
|
+
# 文件格式: { "ip" => { 统一 schema 结果, "ts" => 写入时间戳 } }
|
|
13
|
+
# (与 ngeo-cache.json 同构, 便于 -m 合并与跨机拷贝)
|
|
14
|
+
#
|
|
15
|
+
# 三个角色:
|
|
16
|
+
# 1. 查询源 — lookup(ip) 遍历目录内全部缓存文件, 返回该 IP 的定位信息
|
|
17
|
+
# (目录内文件增删改自动感知, 服务端/客户端均无需重启)
|
|
18
|
+
# 2. 产出 — put_batch(results) 把查询结果合并写入当天的 geocacheYYYYMMDD.json
|
|
19
|
+
# (同一天多次产出自动归并到同一文件, 同 IP 以新结果覆盖)
|
|
20
|
+
# 3. 整理 — merge! 把目录内全部缓存文件合并为一个新的 geocacheYYYYMMDD.json
|
|
21
|
+
# (同 IP 冲突取 ts 最新; 合并生成时间即新文件的时间标签)
|
|
22
|
+
#
|
|
23
|
+
# 命令行入口: bin/ngeo-get (-a 指定目录 / -c 产出 / -nc 不产出 / -m 合并)
|
|
24
|
+
# 服务端入口: bin/geo-api -a (本地库查不到时兜底, 详见 service/geodb/api.rb)
|
|
25
|
+
class GeoCache
|
|
26
|
+
# 缓存文件名: geocache + 8 位日期标签 + .json
|
|
27
|
+
FILE_RE = /\Ageocache\d{8}\.json\z/.freeze
|
|
28
|
+
# -p 顺位参数的合法源
|
|
29
|
+
ORDER_SOURCES = %w[cache local internet].freeze
|
|
30
|
+
# 可缓存状态 (与 GeoQuery::CACHEABLE_STATES 保持一致;
|
|
31
|
+
# 独立定义以支持 geodb 服务端单独加载本文件, 不依赖 geoquery.rb)
|
|
32
|
+
CACHEABLE_STATES = %w[local merged online empty cache].freeze
|
|
33
|
+
# 归属字段白名单 (产出/转换时提取)
|
|
34
|
+
FIELDS = %w[country province city isp asn asn_org network usage].freeze
|
|
35
|
+
|
|
36
|
+
attr_reader :dir
|
|
37
|
+
|
|
38
|
+
# 默认产出目录: ENV["GEO_CACHE_DIR"] || 当前目录
|
|
39
|
+
def self.default_dir
|
|
40
|
+
d = ENV["GEO_CACHE_DIR"].to_s
|
|
41
|
+
d.empty? ? Dir.pwd : File.expand_path(d)
|
|
42
|
+
end
|
|
43
|
+
|
|
44
|
+
# 解析 -p 顺位参数 ("cache,local,internet" / "local internet cache" 等):
|
|
45
|
+
# 返回源名数组; 非法 (含未知源 / 重复 / 为空) 返回 nil
|
|
46
|
+
def self.parse_order(str)
|
|
47
|
+
parts = str.to_s.split(/[,\s]+/).map(&:strip).reject(&:empty?)
|
|
48
|
+
return nil if parts.empty?
|
|
49
|
+
return nil unless parts.all? { |p| ORDER_SOURCES.include?(p) }
|
|
50
|
+
return nil unless parts.uniq.size == parts.size
|
|
51
|
+
parts
|
|
52
|
+
end
|
|
53
|
+
|
|
54
|
+
# 统一 schema → geo-api 接口的 GeoLite2 风格响应 (服务端缓存命中时用)。
|
|
55
|
+
# 对应字段缺失时返回 nil (视作该接口无数据, 由调用方继续下一数据源)。
|
|
56
|
+
def self.to_geolite(kind, hit)
|
|
57
|
+
case kind
|
|
58
|
+
when :asn
|
|
59
|
+
return nil if hit["asn"].to_s.empty?
|
|
60
|
+
{
|
|
61
|
+
"network" => hit["network"].to_s,
|
|
62
|
+
"autonomous_system_number" => hit["asn"].to_s,
|
|
63
|
+
"autonomous_system_organization" => hit["asn_org"].to_s,
|
|
64
|
+
"cached" => true,
|
|
65
|
+
}
|
|
66
|
+
when :city
|
|
67
|
+
return nil if hit["province"].to_s.empty? && hit["city"].to_s.empty?
|
|
68
|
+
g = {}
|
|
69
|
+
g["country_name"] = hit["country"].to_s unless hit["country"].to_s.empty?
|
|
70
|
+
g["subdivision_1_name"] = hit["province"].to_s unless hit["province"].to_s.empty?
|
|
71
|
+
g["city_namezh"] = hit["city"].to_s unless hit["city"].to_s.empty?
|
|
72
|
+
{ "network" => hit["network"].to_s, "geoname" => g, "cached" => true }
|
|
73
|
+
when :country
|
|
74
|
+
return nil if hit["country"].to_s.empty?
|
|
75
|
+
{
|
|
76
|
+
"network" => hit["network"].to_s,
|
|
77
|
+
"geoname" => { "country_name" => hit["country"].to_s },
|
|
78
|
+
"cached" => true,
|
|
79
|
+
}
|
|
80
|
+
end
|
|
81
|
+
end
|
|
82
|
+
|
|
83
|
+
def initialize(dir)
|
|
84
|
+
@dir = File.expand_path(dir.to_s)
|
|
85
|
+
@files = {} # path => { mtime: Float, data: Hash }
|
|
86
|
+
@lock = Mutex.new
|
|
87
|
+
end
|
|
88
|
+
|
|
89
|
+
# 查询: 遍历目录内全部缓存文件, 返回该 IP 的记录 (附加 "cached" => true),
|
|
90
|
+
# 未命中返回 nil。多文件命中时取 ts 最新。
|
|
91
|
+
def lookup(ip)
|
|
92
|
+
@lock.synchronize do
|
|
93
|
+
refresh!
|
|
94
|
+
best = nil
|
|
95
|
+
@files.each_value do |f|
|
|
96
|
+
hit = f[:data][ip]
|
|
97
|
+
next unless hit.is_a?(Hash)
|
|
98
|
+
next if best && hit["ts"].to_i <= best["ts"].to_i
|
|
99
|
+
best = hit
|
|
100
|
+
end
|
|
101
|
+
best ? best.merge("cached" => true) : nil
|
|
102
|
+
end
|
|
103
|
+
end
|
|
104
|
+
|
|
105
|
+
# 产出: 把查询结果 (统一 schema Hash 数组, 或 {ip=>result} Hash)
|
|
106
|
+
# 合并写入当天缓存文件 geocacheYYYYMMDD.json。
|
|
107
|
+
# 仅落盘可缓存状态 (local/merged/online/empty/cache); 同 IP 以本次结果覆盖。
|
|
108
|
+
# 返回 { "file" => 路径, "entries" => 写入条数 }; 无可写内容返回 nil。
|
|
109
|
+
def put_batch(results)
|
|
110
|
+
entries = normalize_entries(results)
|
|
111
|
+
return nil if entries.empty?
|
|
112
|
+
@lock.synchronize do
|
|
113
|
+
path = File.join(@dir, "geocache#{Time.now.strftime('%Y%m%d')}.json")
|
|
114
|
+
data = read_file(path)
|
|
115
|
+
entries.each { |ip, r| data[ip] = r.merge("ts" => Time.now.to_i) }
|
|
116
|
+
write_file(path, data)
|
|
117
|
+
{ "file" => path, "entries" => entries.size }
|
|
118
|
+
end
|
|
119
|
+
end
|
|
120
|
+
|
|
121
|
+
# 整理: 合并目录内全部缓存文件为一个新的 geocacheYYYYMMDD.json
|
|
122
|
+
# (合并生成时间即时间标签; 同 IP 冲突取 ts 最新)。
|
|
123
|
+
# 返回 { "files" => 合并文件数, "entries" => 条目数, "output" => 输出路径 }。
|
|
124
|
+
def merge!
|
|
125
|
+
@lock.synchronize do
|
|
126
|
+
refresh!
|
|
127
|
+
source_count = @files.size # 写入前统计 (输出文件可能覆盖同名源文件)
|
|
128
|
+
merged = {}
|
|
129
|
+
@files.each_value do |f|
|
|
130
|
+
f[:data].each do |ip, r|
|
|
131
|
+
next unless r.is_a?(Hash)
|
|
132
|
+
cur = merged[ip]
|
|
133
|
+
merged[ip] = r if cur.nil? || r["ts"].to_i >= cur["ts"].to_i
|
|
134
|
+
end
|
|
135
|
+
end
|
|
136
|
+
path = File.join(@dir, "geocache#{Time.now.strftime('%Y%m%d')}.json")
|
|
137
|
+
write_file(path, merged)
|
|
138
|
+
{ "files" => source_count, "entries" => merged.size, "output" => path }
|
|
139
|
+
end
|
|
140
|
+
end
|
|
141
|
+
|
|
142
|
+
# 目录统计: { "dir", "files", "entries", "states" }
|
|
143
|
+
def stats
|
|
144
|
+
@lock.synchronize do
|
|
145
|
+
refresh!
|
|
146
|
+
states = Hash.new(0)
|
|
147
|
+
@files.each_value do |f|
|
|
148
|
+
f[:data].each_value { |r| states[r["state"]] += 1 if r.is_a?(Hash) }
|
|
149
|
+
end
|
|
150
|
+
{
|
|
151
|
+
"dir" => @dir,
|
|
152
|
+
"files" => @files.size,
|
|
153
|
+
"entries" => @files.values.sum { |f| f[:data].size },
|
|
154
|
+
"states" => states,
|
|
155
|
+
}
|
|
156
|
+
end
|
|
157
|
+
end
|
|
158
|
+
|
|
159
|
+
private
|
|
160
|
+
|
|
161
|
+
# 目录内全部合法缓存文件 (按文件名排序 = 按日期标签排序)
|
|
162
|
+
def cache_files
|
|
163
|
+
return [] unless File.directory?(@dir)
|
|
164
|
+
Dir.glob(File.join(@dir, "geocache*.json"))
|
|
165
|
+
.select { |p| FILE_RE =~ File.basename(p) }
|
|
166
|
+
.sort
|
|
167
|
+
end
|
|
168
|
+
|
|
169
|
+
# 感知目录变化: 新增/修改的文件重载, 已删除的移除
|
|
170
|
+
def refresh!
|
|
171
|
+
current = {}
|
|
172
|
+
cache_files.each do |path|
|
|
173
|
+
mtime = File.mtime(path).to_f
|
|
174
|
+
prev = @files[path]
|
|
175
|
+
current[path] = if prev && prev[:mtime] == mtime
|
|
176
|
+
prev
|
|
177
|
+
else
|
|
178
|
+
{ mtime: mtime, data: read_file(path) }
|
|
179
|
+
end
|
|
180
|
+
end
|
|
181
|
+
@files.replace(current)
|
|
182
|
+
end
|
|
183
|
+
|
|
184
|
+
def read_file(path)
|
|
185
|
+
return {} unless File.exist?(path)
|
|
186
|
+
raw = File.binread(path)
|
|
187
|
+
raw = raw.delete_prefix("\xEF\xBB\xBF".b) # 容错 UTF-8 BOM (Windows 记事本)
|
|
188
|
+
data = JSON.parse(raw)
|
|
189
|
+
data.is_a?(Hash) ? data : {}
|
|
190
|
+
rescue JSON::ParserError, Errno::ENOENT
|
|
191
|
+
{}
|
|
192
|
+
end
|
|
193
|
+
|
|
194
|
+
def write_file(path, data)
|
|
195
|
+
require "fileutils"
|
|
196
|
+
FileUtils.mkdir_p(@dir)
|
|
197
|
+
tmp = "#{path}.tmp#{Process.pid}"
|
|
198
|
+
File.binwrite(tmp, JSON.pretty_generate(data))
|
|
199
|
+
File.rename(tmp, path)
|
|
200
|
+
@files[path] = { mtime: File.mtime(path).to_f, data: data }
|
|
201
|
+
path
|
|
202
|
+
end
|
|
203
|
+
|
|
204
|
+
# 产出内容规整: Hash 数组 (取 ip 字段) 或 {ip=>result} Hash
|
|
205
|
+
# 仅保留 ip 合法且状态可缓存的条目
|
|
206
|
+
def normalize_entries(results)
|
|
207
|
+
entries = {}
|
|
208
|
+
if results.is_a?(Hash)
|
|
209
|
+
results.each do |ip, r|
|
|
210
|
+
next unless r.is_a?(Hash) && cacheable?(r) && GeoQuery.valid_ip?(ip)
|
|
211
|
+
entries[ip.to_s] = pick_fields(r)
|
|
212
|
+
end
|
|
213
|
+
elsif results.is_a?(Array)
|
|
214
|
+
results.each do |r|
|
|
215
|
+
next unless r.is_a?(Hash)
|
|
216
|
+
ip = r["ip"].to_s
|
|
217
|
+
next unless cacheable?(r) && GeoQuery.valid_ip?(ip)
|
|
218
|
+
entries[ip] = pick_fields(r)
|
|
219
|
+
end
|
|
220
|
+
end
|
|
221
|
+
entries
|
|
222
|
+
end
|
|
223
|
+
|
|
224
|
+
def cacheable?(result)
|
|
225
|
+
CACHEABLE_STATES.include?(result["state"].to_s)
|
|
226
|
+
end
|
|
227
|
+
|
|
228
|
+
# 提取归属字段 (剔除 state/message/source/sources/ts/cached 等元数据)
|
|
229
|
+
def pick_fields(r)
|
|
230
|
+
r.select { |k, _| FIELDS.include?(k) }
|
|
231
|
+
end
|
|
232
|
+
end
|
|
233
|
+
end
|
|
@@ -0,0 +1,327 @@
|
|
|
1
|
+
# coding: utf-8
|
|
2
|
+
# frozen_string_literal: true
|
|
3
|
+
|
|
4
|
+
# GeoQuery — IP 归属综合查询服务 (GEO_CACHE 缓存 + 本地 geo-api + 互联网 ip-api.com)
|
|
5
|
+
#
|
|
6
|
+
# 命令行入口: bin/gen-get (纯互联网) / bin/ngeo-get (缓存+本地+互联网综合)
|
|
7
|
+
# 详见同目录 GeoQuery.md。
|
|
8
|
+
#
|
|
9
|
+
# 查询模型 (ngeo, 三源顺位编排 — 顺位即优先级, 满意即停, 字段叠加):
|
|
10
|
+
#
|
|
11
|
+
# 1. 会话缓存 (ngeo-cache.json) 命中 → 直接返回
|
|
12
|
+
# 2. 按 -p 顺位逐源查询 (默认 cache → local → internet):
|
|
13
|
+
# - cache GEO_CACHE 外带缓存目录 (geocacheXXXXXXXX.json, bin/ngeo-get -a)
|
|
14
|
+
# - local 本地 geo-api 服务 (GeoLite2)
|
|
15
|
+
# - internet 互联网 ip-api.com (限速 + 熔断保护)
|
|
16
|
+
# 3. 每源结果按顺位折叠: 先查的源字段优先, 后查的补空缺
|
|
17
|
+
# 4. 折叠后归属满意 (省/市/ASN 齐全) → 提前终止, 不再查后续源
|
|
18
|
+
# 5. 全部源查完仍不满意 → 组装终态 (叠加/部分/空/不可达)
|
|
19
|
+
#
|
|
20
|
+
# 结果状态 (state) 按可信度排序 — 体现 空结果 < 部分 < 缓存 < 实时 < 叠加:
|
|
21
|
+
#
|
|
22
|
+
# unreachable 各源均无结果且有不联通 (空结果状态, 最低)
|
|
23
|
+
# local-partial 本地部分结果, 无更多补充 (保留本地字段)
|
|
24
|
+
# cache-partial 缓存部分结果, 无更多补充 (保留缓存字段)
|
|
25
|
+
# empty 各源均确认无归属 (如私有/保留地址)
|
|
26
|
+
# cache 缓存命中且满意
|
|
27
|
+
# local 本地结果满意 (省/市/ASN 齐全)
|
|
28
|
+
# online 纯互联网结果
|
|
29
|
+
# merged 多源叠加综合 (最高)
|
|
30
|
+
module GeoQuery
|
|
31
|
+
autoload :Cache, File.expand_path("cache", __dir__)
|
|
32
|
+
autoload :GeoCache, File.expand_path("geo_cache", __dir__)
|
|
33
|
+
autoload :LocalClient, File.expand_path("local", __dir__)
|
|
34
|
+
autoload :Merge, File.expand_path("merge", __dir__)
|
|
35
|
+
autoload :Normalize, File.expand_path("normalize", __dir__)
|
|
36
|
+
autoload :OnlineClient, File.expand_path("online", __dir__)
|
|
37
|
+
end
|
|
38
|
+
|
|
39
|
+
module GeoQuery
|
|
40
|
+
# 可缓存的结果状态 (unreachable / local-partial / cache-partial 不落盘,
|
|
41
|
+
# 下次重查以便互联网恢复后自动补全)
|
|
42
|
+
CACHEABLE_STATES = %w[local merged online empty cache].freeze
|
|
43
|
+
end
|
|
44
|
+
|
|
45
|
+
class GeoQuery::NGeo
|
|
46
|
+
# 满意度判定字段: 省份 / 城市 / ASN (ASN 齐则用途可推断)
|
|
47
|
+
SATISFY_FIELDS = %w[province city asn].freeze
|
|
48
|
+
|
|
49
|
+
# 互联网熔断: 连续失败 N 次后, COOLDOWN 秒内跳过在线查询
|
|
50
|
+
BREAKER_THRESHOLD = 3
|
|
51
|
+
BREAKER_COOLDOWN = 60
|
|
52
|
+
|
|
53
|
+
# 默认查询顺位: 缓存 → 本地 → 互联网
|
|
54
|
+
DEFAULT_ORDER = %w[cache local internet].freeze
|
|
55
|
+
# 合法顺位源 (与 GeoQuery::GeoCache::ORDER_SOURCES 一致)
|
|
56
|
+
ORDER_SOURCES = %w[cache local internet].freeze
|
|
57
|
+
# sources 键 → 结果来源标注
|
|
58
|
+
SRC_LABEL = {
|
|
59
|
+
"cache" => "geo-cache", # GEO_CACHE 外带缓存
|
|
60
|
+
"local" => "geo-get", # 本地 geo-api
|
|
61
|
+
"online" => "gen-get", # 互联网 ip-api.com
|
|
62
|
+
}.freeze
|
|
63
|
+
|
|
64
|
+
attr_reader :local, :online, :cache, :geo_cache, :order
|
|
65
|
+
|
|
66
|
+
def initialize(geoapi_base: nil, api: nil, timeout: 10,
|
|
67
|
+
cache_dir: nil, cache_file: nil,
|
|
68
|
+
geo_cache_dir: nil, order: DEFAULT_ORDER)
|
|
69
|
+
base = geoapi_base || ENV["GEO_API_BASE"] || GeoQuery::LocalClient::DEFAULT_BASE
|
|
70
|
+
@local = GeoQuery::LocalClient.new(base: base)
|
|
71
|
+
@online = GeoQuery::OnlineClient.new(api: api, timeout: timeout)
|
|
72
|
+
dir = cache_dir || GeoQuery::Cache.default_dir
|
|
73
|
+
@cache = GeoQuery::Cache.new(cache_file || File.join(dir, "ngeo-cache.json"))
|
|
74
|
+
@geo_cache = geo_cache_dir ? GeoQuery::GeoCache.new(geo_cache_dir) : nil
|
|
75
|
+
@order = normalize_order(order)
|
|
76
|
+
@breaker_mutex = Mutex.new
|
|
77
|
+
@failures = 0
|
|
78
|
+
@breaker_until = nil
|
|
79
|
+
end
|
|
80
|
+
|
|
81
|
+
# 综合查询单个 IP。
|
|
82
|
+
# refresh: 忽略缓存强制重查 (会话缓存与 GEO_CACHE 均跳过)
|
|
83
|
+
# no_local: 跳过本地 geo-api
|
|
84
|
+
# save: 是否把结果写入会话缓存 (默认 true)
|
|
85
|
+
# 查询产出 (geocacheYYYYMMDD.json) 由调用方用 GeoQuery::GeoCache#put_batch 落盘。
|
|
86
|
+
def lookup(ip, refresh: false, no_local: false, save: true)
|
|
87
|
+
ip = ip.to_s.strip
|
|
88
|
+
return invalid_result(ip, "IP 不合法") unless GeoQuery.valid_ip?(ip)
|
|
89
|
+
|
|
90
|
+
hit = @cache.get(ip) unless refresh
|
|
91
|
+
return hit if hit
|
|
92
|
+
|
|
93
|
+
result = chain_lookup(ip, refresh: refresh, no_local: no_local)
|
|
94
|
+
@cache.put(ip, result) if save && GeoQuery::CACHEABLE_STATES.include?(result["state"])
|
|
95
|
+
result
|
|
96
|
+
end
|
|
97
|
+
|
|
98
|
+
def save_cache
|
|
99
|
+
@cache.save
|
|
100
|
+
end
|
|
101
|
+
|
|
102
|
+
private
|
|
103
|
+
|
|
104
|
+
# ---- 三源顺位编排 ---------------------------------------------------------
|
|
105
|
+
|
|
106
|
+
def chain_lookup(ip, refresh: false, no_local: false)
|
|
107
|
+
plan = planned_order(refresh: refresh, no_local: no_local)
|
|
108
|
+
acc = nil
|
|
109
|
+
sources = {}
|
|
110
|
+
|
|
111
|
+
# 不参与本次查询的源, 仍按原输出契约标注
|
|
112
|
+
if no_local && @order.include?("local")
|
|
113
|
+
sources["local"] = { "state" => "unavailable", "message" => "本地查询已跳过 (--no-local)" }
|
|
114
|
+
end
|
|
115
|
+
if refresh && @geo_cache && @order.include?("cache")
|
|
116
|
+
sources["cache"] = { "state" => "skipped", "message" => "已忽略缓存强制重查 (--refresh)" }
|
|
117
|
+
end
|
|
118
|
+
|
|
119
|
+
plan.each do |src|
|
|
120
|
+
r = query_source(src, ip)
|
|
121
|
+
key = src_key(src)
|
|
122
|
+
sources[key] = note_for(key, r)
|
|
123
|
+
next unless r["state"] == "ok"
|
|
124
|
+
acc = fold(acc, r)
|
|
125
|
+
break if satisfied_fields?(acc)
|
|
126
|
+
end
|
|
127
|
+
|
|
128
|
+
# 满意提前终止后, 顺位中剩余未查的源 → skipped
|
|
129
|
+
plan.each do |src|
|
|
130
|
+
key = src_key(src)
|
|
131
|
+
sources[key] = { "state" => "skipped" } unless sources.key?(key)
|
|
132
|
+
end
|
|
133
|
+
|
|
134
|
+
assemble(ip, acc, sources)
|
|
135
|
+
end
|
|
136
|
+
|
|
137
|
+
# 实际查询的源列表: 顺位中剔除不可用源
|
|
138
|
+
# (无 GEO_CACHE 目录 → cache 源不存在; refresh → 跳过缓存; no_local → 跳过本地)
|
|
139
|
+
def planned_order(refresh: false, no_local: false)
|
|
140
|
+
@order.reject do |src|
|
|
141
|
+
(src == "cache" && (@geo_cache.nil? || refresh)) ||
|
|
142
|
+
(src == "local" && no_local)
|
|
143
|
+
end
|
|
144
|
+
end
|
|
145
|
+
|
|
146
|
+
def query_source(src, ip)
|
|
147
|
+
case src
|
|
148
|
+
when "cache" then query_cache(ip)
|
|
149
|
+
when "local" then @local.lookup(ip)
|
|
150
|
+
when "internet" then online_with_breaker { @online.lookup(ip) }
|
|
151
|
+
end
|
|
152
|
+
end
|
|
153
|
+
|
|
154
|
+
# GEO_CACHE 缓存查询: 命中记录保留产出时的归属字段,
|
|
155
|
+
# 有归属 → 规整为 ok 参与折叠; empty → 确认无归属; 未命中 → miss (继续下一源)
|
|
156
|
+
def query_cache(ip)
|
|
157
|
+
hit = @geo_cache ? @geo_cache.lookup(ip) : nil
|
|
158
|
+
return cache_base(ip, "miss", "GEO_CACHE 缓存未命中") unless hit
|
|
159
|
+
if hit["state"] == "empty"
|
|
160
|
+
cache_base(ip, "empty", "缓存确认无归属")
|
|
161
|
+
else
|
|
162
|
+
cache_base(ip, "ok", "").merge(pick_fields(hit))
|
|
163
|
+
end
|
|
164
|
+
end
|
|
165
|
+
|
|
166
|
+
def cache_base(ip, state, message)
|
|
167
|
+
{
|
|
168
|
+
"ip" => ip, "state" => state, "message" => message,
|
|
169
|
+
"country" => "", "province" => "", "city" => "",
|
|
170
|
+
"isp" => "", "asn" => "", "asn_org" => "", "network" => "",
|
|
171
|
+
"usage" => "", "source" => "geo-cache", "cached" => true,
|
|
172
|
+
}
|
|
173
|
+
end
|
|
174
|
+
|
|
175
|
+
# 从缓存记录提取归属字段 (剔除 state/message/source/sources/ts/cached)
|
|
176
|
+
def pick_fields(hit)
|
|
177
|
+
hit.select { |k, _| GeoQuery::GeoCache::FIELDS.include?(k) }
|
|
178
|
+
end
|
|
179
|
+
|
|
180
|
+
# ---- 折叠 / 终态组装 ------------------------------------------------------
|
|
181
|
+
|
|
182
|
+
# acc 优先, r 补空缺 (顺位即优先级)
|
|
183
|
+
def fold(acc, r)
|
|
184
|
+
return r.dup unless acc
|
|
185
|
+
GeoQuery::Merge.fields(acc, r)
|
|
186
|
+
end
|
|
187
|
+
|
|
188
|
+
def assemble(ip, acc, sources)
|
|
189
|
+
ok_srcs = sources.select { |_, n| n["state"] == "ok" }.keys
|
|
190
|
+
fields = acc || {}
|
|
191
|
+
|
|
192
|
+
state =
|
|
193
|
+
if ok_srcs.empty?
|
|
194
|
+
# 无任何归属字段: 有源确认无归属且无不联通 → empty, 否则 unreachable
|
|
195
|
+
has_empty = sources.values.any? { |n| n["state"] == "empty" }
|
|
196
|
+
net_down = sources.values.any? { |n| n["state"] == "unreachable" }
|
|
197
|
+
(has_empty && !net_down) ? "empty" : "unreachable"
|
|
198
|
+
elsif ok_srcs.size == 1
|
|
199
|
+
s = ok_srcs.first
|
|
200
|
+
if satisfied_fields?(fields)
|
|
201
|
+
{ "cache" => "cache", "local" => "local", "online" => "online" }.fetch(s, "merged")
|
|
202
|
+
elsif s == "local"
|
|
203
|
+
"local-partial"
|
|
204
|
+
elsif s == "cache"
|
|
205
|
+
"cache-partial"
|
|
206
|
+
else
|
|
207
|
+
"online" # 互联网结果即使不满意仍为 online (对齐原模型)
|
|
208
|
+
end
|
|
209
|
+
else
|
|
210
|
+
"merged"
|
|
211
|
+
end
|
|
212
|
+
|
|
213
|
+
{
|
|
214
|
+
"ip" => ip,
|
|
215
|
+
"state" => state,
|
|
216
|
+
"message" => state_message(state, sources),
|
|
217
|
+
"country" => fields["country"].to_s,
|
|
218
|
+
"province" => fields["province"].to_s,
|
|
219
|
+
"city" => fields["city"].to_s,
|
|
220
|
+
"isp" => GeoQuery::Normalize.isp_from_asn(fields["asn_org"].to_s),
|
|
221
|
+
"asn" => fields["asn"].to_s,
|
|
222
|
+
"asn_org" => fields["asn_org"].to_s,
|
|
223
|
+
"network" => fields["network"].to_s,
|
|
224
|
+
"usage" => fields["usage"].to_s,
|
|
225
|
+
"source" => ok_srcs.empty? ? "ngeo" : "ngeo(#{ok_srcs.map { |s| SRC_LABEL[s] }.join('+')})",
|
|
226
|
+
"sources" => sources,
|
|
227
|
+
}.tap do |r|
|
|
228
|
+
r["cached"] = true if ok_srcs.include?("cache")
|
|
229
|
+
end
|
|
230
|
+
end
|
|
231
|
+
|
|
232
|
+
# 各源查询说明 → 终态 message
|
|
233
|
+
def state_message(state, sources)
|
|
234
|
+
online = sources["online"]
|
|
235
|
+
case state
|
|
236
|
+
when "unreachable"
|
|
237
|
+
if online && online["state"] == "unreachable"
|
|
238
|
+
"互联网查询不可达: #{online['message']}"
|
|
239
|
+
else
|
|
240
|
+
"各数据源均无结果 (本地无数据/GEO_CACHE 未命中)"
|
|
241
|
+
end
|
|
242
|
+
when "empty"
|
|
243
|
+
if online && online["state"] == "empty"
|
|
244
|
+
"各数据源均确认无归属 (#{online['message']})"
|
|
245
|
+
else
|
|
246
|
+
"各数据源均确认无归属"
|
|
247
|
+
end
|
|
248
|
+
when "local-partial", "cache-partial"
|
|
249
|
+
if online && online["state"] == "unreachable"
|
|
250
|
+
"互联网查询不可达: #{online['message']}"
|
|
251
|
+
elsif online && online["state"] == "empty"
|
|
252
|
+
"互联网确认无归属 (#{online['message']})"
|
|
253
|
+
else
|
|
254
|
+
"其余数据源无补充结果"
|
|
255
|
+
end
|
|
256
|
+
else
|
|
257
|
+
""
|
|
258
|
+
end
|
|
259
|
+
end
|
|
260
|
+
|
|
261
|
+
def note_for(key, client_result)
|
|
262
|
+
note = { "state" => client_result["state"] }
|
|
263
|
+
unless client_result["message"].to_s.empty?
|
|
264
|
+
note["message"] = client_result["message"]
|
|
265
|
+
end
|
|
266
|
+
if client_result["state"] == "ok" && key != "online"
|
|
267
|
+
note["verdict"] = satisfied_fields?(client_result) ? "满意" : "不满意"
|
|
268
|
+
end
|
|
269
|
+
note
|
|
270
|
+
end
|
|
271
|
+
|
|
272
|
+
def invalid_result(ip, message)
|
|
273
|
+
{
|
|
274
|
+
"ip" => ip, "state" => "invalid", "message" => message,
|
|
275
|
+
"country" => "", "province" => "", "city" => "",
|
|
276
|
+
"isp" => "", "asn" => "", "asn_org" => "", "network" => "",
|
|
277
|
+
"usage" => "", "source" => "ngeo", "sources" => {},
|
|
278
|
+
}
|
|
279
|
+
end
|
|
280
|
+
|
|
281
|
+
# ---- 满意度 / 熔断 --------------------------------------------------------
|
|
282
|
+
|
|
283
|
+
# 满意 = 省份、城市、ASN 全部非空 (ASN 有则用途可推断)
|
|
284
|
+
def satisfied_fields?(result)
|
|
285
|
+
SATISFY_FIELDS.all? { |f| !result[f].to_s.empty? }
|
|
286
|
+
end
|
|
287
|
+
|
|
288
|
+
# 顺位参数规整: String (经 GeoCache.parse_order) 或 Array;
|
|
289
|
+
# 非法时回退默认顺位
|
|
290
|
+
def normalize_order(order)
|
|
291
|
+
parts = order.is_a?(String) ? GeoQuery::GeoCache.parse_order(order) : order
|
|
292
|
+
unless parts.is_a?(Array) && !parts.empty?
|
|
293
|
+
parts = DEFAULT_ORDER
|
|
294
|
+
end
|
|
295
|
+
parts.select { |s| ORDER_SOURCES.include?(s) }
|
|
296
|
+
end
|
|
297
|
+
|
|
298
|
+
# 互联网熔断: 连续 3 次不可达后 60 秒内直接返回 unreachable, 不再发包
|
|
299
|
+
def online_with_breaker
|
|
300
|
+
@breaker_mutex.synchronize do
|
|
301
|
+
now = Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
|
302
|
+
if @breaker_until && now < @breaker_until
|
|
303
|
+
return {
|
|
304
|
+
"state" => "unreachable",
|
|
305
|
+
"message" => "熔断中 (前 #{@failures_total} 次不可达, 暂停在线查询)",
|
|
306
|
+
}
|
|
307
|
+
end
|
|
308
|
+
result = yield
|
|
309
|
+
if result["state"] == "unreachable"
|
|
310
|
+
@failures += 1
|
|
311
|
+
if @failures >= BREAKER_THRESHOLD
|
|
312
|
+
@failures_total = BREAKER_THRESHOLD
|
|
313
|
+
@breaker_until = now + BREAKER_COOLDOWN
|
|
314
|
+
@failures = 0
|
|
315
|
+
end
|
|
316
|
+
else
|
|
317
|
+
@failures = 0
|
|
318
|
+
end
|
|
319
|
+
result
|
|
320
|
+
end
|
|
321
|
+
end
|
|
322
|
+
|
|
323
|
+
# sources 键名: internet 源 → "online" (沿用既有输出契约)
|
|
324
|
+
def src_key(src)
|
|
325
|
+
src == "internet" ? "online" : src
|
|
326
|
+
end
|
|
327
|
+
end
|