net-connector 0.4.1 → 0.4.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +34 -0
- data/CONTRIBUTING.md +40 -0
- data/README.md +61 -9
- data/SECURITY.md +11 -0
- data/docs/VERIFICATION.md +129 -6
- data/docs/architecture.md +184 -7
- data/lib/net/connector/device/base.rb +11 -4
- data/lib/net/connector/device/running_config.rb +2 -0
- data/lib/net/connector/engine/command.rb +14 -2
- data/lib/net/connector/engine/configuration.rb +10 -7
- data/lib/net/connector/engine/dialogue.rb +52 -28
- data/lib/net/connector/engine/errors.rb +40 -55
- data/lib/net/connector/engine/execution.rb +32 -5
- data/lib/net/connector/engine/log.rb +11 -9
- data/lib/net/connector/engine/session.rb +38 -7
- data/lib/net/connector/engine/terminal_renderer.rb +7 -4
- data/lib/net/connector/netdisco/batch.rb +31 -3
- data/lib/net/connector/netdisco/cli.rb +73 -31
- data/lib/net/connector/netdisco/client.rb +206 -73
- data/lib/net/connector/netdisco/config_file.rb +26 -4
- data/lib/net/connector/netdisco/diagnostic.rb +91 -0
- data/lib/net/connector/netdisco/fleet.rb +103 -60
- data/lib/net/connector/netdisco/inventory_budget.rb +49 -0
- data/lib/net/connector/netdisco/report.rb +94 -0
- data/lib/net/connector/netdisco/rules.rb +28 -5
- data/lib/net/connector/netdisco/settings.rb +194 -85
- data/lib/net/connector/netdisco/worker.rb +26 -17
- data/lib/net/connector/operations/backup_lock.rb +116 -0
- data/lib/net/connector/operations/local_backup.rb +49 -9
- data/lib/net/connector/operations/parse_output.rb +20 -3
- data/lib/net/connector/operations/private_file.rb +94 -5
- data/lib/net/connector/operations/safe_file.rb +62 -0
- data/lib/net/connector/operations/saved_config/legacy_index.rb +109 -0
- data/lib/net/connector/operations/saved_config.rb +40 -10
- data/lib/net/connector/operations/tftp/file_upload.rb +14 -1
- data/lib/net/connector/operations/tftp_backup.rb +78 -15
- data/lib/net/connector/operations/tftp_receipt.rb +73 -0
- data/lib/net/connector/operations/topology/immediate_strategy.rb +45 -0
- data/lib/net/connector/operations/topology/strategy.rb +20 -0
- data/lib/net/connector/operations/topology.rb +97 -29
- data/lib/net/connector/operations.rb +6 -0
- data/lib/net/connector/vendor/cisco_ios/tftp_backup.rb +9 -2
- data/lib/net/connector/vendor/cisco_ios/topology.rb +10 -4
- data/lib/net/connector/vendor/cisco_nxos/tftp_backup.rb +8 -1
- data/lib/net/connector/vendor/h3c/tftp_backup.rb +5 -0
- data/lib/net/connector/vendor/h3c/topology.rb +9 -4
- data/lib/net/connector/vendor/hillstone/tftp_backup.rb +12 -3
- data/lib/net/connector/vendor/hillstone/topology.rb +9 -4
- data/lib/net/connector/vendor/huawei/tftp_backup.rb +6 -0
- data/lib/net/connector/vendor/palo_alto/tftp_backup.rb +14 -5
- data/lib/net/connector/vendor/palo_alto/topology.rb +7 -2
- data/lib/net/connector/vendor/radware/tftp_backup.rb +9 -2
- data/lib/net/connector/vendor/radware/topology.rb +1 -0
- data/lib/net/connector/version.rb +1 -1
- metadata +35 -5
|
@@ -1,41 +1,95 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
3
|
require "json"
|
|
4
|
-
require "uri"
|
|
5
4
|
|
|
6
5
|
module Net
|
|
7
6
|
module Connector
|
|
8
7
|
module Netdisco
|
|
9
|
-
#
|
|
8
|
+
# 活的凭据来源与不可变的批次策略分开保存;只有明确列出的非敏感字段进入快照。
|
|
10
9
|
class Settings
|
|
11
10
|
DEFAULT_BACKUP_DIRECTORY = "backups"
|
|
12
11
|
DEFAULT_CONCURRENCY = 4
|
|
13
12
|
MAX_CONCURRENCY = 50
|
|
14
|
-
DEFAULT_PAGE_SIZE =
|
|
13
|
+
DEFAULT_PAGE_SIZE = Client::DEFAULTS.fetch(:page_size)
|
|
14
|
+
UNSET = Object.new.freeze
|
|
15
|
+
private_constant :UNSET
|
|
15
16
|
|
|
16
|
-
# 加载 YAML 配置,并使环境变量覆盖文件配置。
|
|
17
17
|
def self.from_file(path, env: ENV)
|
|
18
|
-
new(env: ConfigFile.load(path)
|
|
18
|
+
new(env: env, defaults: ConfigFile.load(path))
|
|
19
19
|
end
|
|
20
20
|
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
@
|
|
21
|
+
def initialize(env: ENV, defaults: {}, overrides: {}, policy: nil, fixed: false)
|
|
22
|
+
@env, @defaults, @overrides = env, defaults, overrides
|
|
23
|
+
@policy, @fixed = policy, fixed
|
|
24
24
|
end
|
|
25
25
|
|
|
26
|
-
#
|
|
27
|
-
def
|
|
28
|
-
|
|
29
|
-
if
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
26
|
+
# 快照自身不保留 ENV、凭据 resolver 或原 Settings 的引用,不能携带秘密进入摘要。
|
|
27
|
+
def snapshot(mode: :backup)
|
|
28
|
+
return @policy.validate!(mode: mode) if @policy
|
|
29
|
+
return validate!(mode: mode) if @fixed
|
|
30
|
+
|
|
31
|
+
keys = mode == :export ? ["NET_CONNECTOR_BACKUP_DIRECTORY"] : non_secret_keys
|
|
32
|
+
values = keys.each_with_object({}) do |key, result|
|
|
33
|
+
item = raw(key, UNSET)
|
|
34
|
+
result[key.freeze] = item.is_a?(String) ? item.dup.freeze : item unless item.equal?(UNSET)
|
|
35
|
+
end
|
|
36
|
+
self.class.new(env: values.freeze, fixed: true).validate!(mode: mode).freeze
|
|
37
|
+
end
|
|
38
|
+
|
|
39
|
+
# CLI 的一次预览/执行使用同一策略;凭据仍指向原环境映射,允许逐设备轮换。
|
|
40
|
+
def for_run(mode: :backup)
|
|
41
|
+
self.class.new(env: @env, defaults: @defaults, overrides: @overrides, policy: snapshot(mode: mode))
|
|
42
|
+
end
|
|
43
|
+
|
|
44
|
+
def validate!(mode:)
|
|
45
|
+
unless %i[backup tftp inventory show_config export].include?(mode)
|
|
46
|
+
raise ArgumentError, "unknown settings validation mode"
|
|
47
|
+
end
|
|
48
|
+
backup_directory
|
|
49
|
+
return self if mode == :export
|
|
50
|
+
|
|
51
|
+
client_options
|
|
52
|
+
rules
|
|
53
|
+
concurrency
|
|
54
|
+
limit_per_vendor
|
|
55
|
+
log_directory
|
|
56
|
+
connection_options
|
|
57
|
+
Net::Connector.vendors.each { |vendor| connection_options(vendor) }
|
|
58
|
+
tftp_source_files.each_value { |source| TftpTarget.validate_source_file!(source) }
|
|
59
|
+
tftp_vrfs
|
|
60
|
+
TftpTarget.new(host: tftp_server, path: "preflight.cfg") if tftp_server
|
|
61
|
+
self
|
|
62
|
+
end
|
|
63
|
+
|
|
64
|
+
def client(policy: snapshot(mode: :inventory))
|
|
65
|
+
options = policy.client_options
|
|
66
|
+
raise ArgumentError, "NETDISCO_URL is required" unless options.fetch(:url)
|
|
67
|
+
|
|
68
|
+
authentication = if (token = secret_value("NETDISCO_API_KEY"))
|
|
69
|
+
{ api_key: token }
|
|
70
|
+
else
|
|
71
|
+
{ username: required_secret("NETDISCO_USERNAME"), password: required_secret("NETDISCO_PASSWORD") }
|
|
72
|
+
end
|
|
73
|
+
Client.new(**options, **authentication)
|
|
74
|
+
end
|
|
75
|
+
|
|
76
|
+
# 原生客户端和 --show-config 共享同一预算、地址与 HTTP 策略校验。
|
|
77
|
+
def client_options
|
|
78
|
+
options = Client::DEFAULTS.to_h do |name, default|
|
|
79
|
+
key = "NETDISCO_#{name.to_s.upcase}"
|
|
80
|
+
parsed = case name
|
|
81
|
+
when :inventory_timeout then number(key, default)
|
|
82
|
+
when :allow_insecure_http then boolean(key, default)
|
|
83
|
+
else integer(key, default)
|
|
84
|
+
end
|
|
85
|
+
[name, parsed]
|
|
34
86
|
end
|
|
35
|
-
Client.
|
|
87
|
+
options = Client.options(**options)
|
|
88
|
+
url = value("NETDISCO_URL")
|
|
89
|
+
Client.validate_url!(url, allow_insecure_http: options.fetch(:allow_insecure_http)) if url
|
|
90
|
+
options.merge(url: url).freeze
|
|
36
91
|
end
|
|
37
92
|
|
|
38
|
-
# 读取清单筛选和厂商映射规则。
|
|
39
93
|
def rules
|
|
40
94
|
Rules.new(include_hosts: list("NET_CONNECTOR_INCLUDE_HOSTS"),
|
|
41
95
|
exclude_hosts: list("NET_CONNECTOR_EXCLUDE_HOSTS"),
|
|
@@ -45,15 +99,13 @@ module Net
|
|
|
45
99
|
mappings: json("NET_CONNECTOR_DEVICE_RULES", []))
|
|
46
100
|
end
|
|
47
101
|
|
|
48
|
-
# 读取并展开本地备份目录。
|
|
49
102
|
def backup_directory
|
|
50
|
-
path =
|
|
103
|
+
path = raw("NET_CONNECTOR_BACKUP_DIRECTORY", DEFAULT_BACKUP_DIRECTORY)
|
|
51
104
|
raise ArgumentError, "NET_CONNECTOR_BACKUP_DIRECTORY must be nonempty" unless present?(path)
|
|
52
105
|
|
|
53
106
|
File.expand_path(path)
|
|
54
107
|
end
|
|
55
108
|
|
|
56
|
-
# 读取并限制设备任务并发数。
|
|
57
109
|
def concurrency
|
|
58
110
|
count = integer("NET_CONNECTOR_CONCURRENCY", DEFAULT_CONCURRENCY)
|
|
59
111
|
raise ArgumentError, "NET_CONNECTOR_CONCURRENCY must be at most #{MAX_CONCURRENCY}" if count > MAX_CONCURRENCY
|
|
@@ -61,121 +113,178 @@ module Net
|
|
|
61
113
|
count
|
|
62
114
|
end
|
|
63
115
|
|
|
64
|
-
# 确定厂商采样数量:本地默认全量,TFTP 默认每厂商五台。
|
|
65
116
|
def limit_per_vendor(tftp: false)
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
117
|
+
count = if configured?("NET_CONNECTOR_SAMPLE_PER_VENDOR")
|
|
118
|
+
integer("NET_CONNECTOR_SAMPLE_PER_VENDOR", 5) unless raw("NET_CONNECTOR_SAMPLE_PER_VENDOR").nil?
|
|
119
|
+
elsif tftp
|
|
120
|
+
5
|
|
121
|
+
end
|
|
122
|
+
Planner.validate_limit!(count)
|
|
123
|
+
count
|
|
69
124
|
end
|
|
70
125
|
|
|
71
|
-
# 读取厂商的 TFTP 源文件配置,并提供华为启动配置默认路径。
|
|
72
126
|
def tftp_source_files
|
|
73
|
-
files = { huawei:
|
|
127
|
+
files = { huawei: raw("NET_CONNECTOR_HUAWEI_TFTP_SOURCE_FILE", "flash:/startup.cfg") }
|
|
74
128
|
%i[h3c h3c_wireless].each do |vendor|
|
|
75
129
|
key = "NET_CONNECTOR_#{vendor.to_s.upcase}_TFTP_SOURCE_FILE"
|
|
76
|
-
files[vendor] =
|
|
130
|
+
files[vendor] = raw(key) if configured?(key)
|
|
77
131
|
end
|
|
78
132
|
files
|
|
79
133
|
end
|
|
80
134
|
|
|
81
|
-
# 读取各厂商 TFTP 出口 VRF 的名称映射。
|
|
82
135
|
def tftp_vrfs
|
|
83
|
-
|
|
84
|
-
raise ArgumentError, "NET_CONNECTOR_TFTP_VRFS must be a JSON object" unless
|
|
136
|
+
mapping = json("NET_CONNECTOR_TFTP_VRFS", {})
|
|
137
|
+
raise ArgumentError, "NET_CONNECTOR_TFTP_VRFS must be a JSON object" unless mapping.is_a?(Hash)
|
|
138
|
+
|
|
139
|
+
self.class.validate_vrfs!(mapping.transform_keys(&:to_sym))
|
|
140
|
+
end
|
|
141
|
+
|
|
142
|
+
def self.validate_vrfs!(vrfs)
|
|
143
|
+
unless vrfs.is_a?(Hash) && vrfs.all? { |vendor, name|
|
|
144
|
+
%i[cisco_nxos hillstone].include?(vendor) && name.is_a?(String) &&
|
|
145
|
+
name.match?(/\A[A-Za-z0-9_][A-Za-z0-9_.-]*\z/)
|
|
146
|
+
}
|
|
147
|
+
raise ArgumentError, "vrfs must map supported vendor names to safe VRF names"
|
|
148
|
+
end
|
|
149
|
+
vrfs
|
|
150
|
+
end
|
|
85
151
|
|
|
86
|
-
|
|
152
|
+
def tftp_server = value("TFTP_HOST")
|
|
153
|
+
|
|
154
|
+
def connection_options(vendor = nil)
|
|
155
|
+
prefix = "NET_CONNECTOR_#{vendor.to_s.upcase}_"
|
|
156
|
+
configuration = Configuration.new(
|
|
157
|
+
protocol: (vendor && value(prefix + "PROTOCOL")) || value("NET_CONNECTOR_PROTOCOL") || "ssh",
|
|
158
|
+
log_level: value("NET_CONNECTOR_LOG_LEVEL") || "info", known_hosts: value("NET_CONNECTOR_KNOWN_HOSTS"),
|
|
159
|
+
host_key_policy: value("NET_CONNECTOR_HOST_KEY_POLICY") || "strict",
|
|
160
|
+
max_script_output_bytes: optional_integer("NET_CONNECTOR_MAX_SCRIPT_OUTPUT_BYTES")
|
|
161
|
+
)
|
|
162
|
+
options = %i[protocol log_level known_hosts host_key_policy].to_h { |key| [key, configuration.public_send(key)] }
|
|
163
|
+
options[:max_script_output_bytes] = configuration.max_script_output_bytes if configuration.max_script_output_bytes
|
|
164
|
+
options.freeze
|
|
87
165
|
end
|
|
88
166
|
|
|
89
|
-
#
|
|
90
|
-
|
|
91
|
-
|
|
167
|
+
# 兼容原入口;Fleet 传入本批策略,独立调用则读取当前策略和当前秘密。
|
|
168
|
+
def credentials_for(device, policy: snapshot(mode: :backup))
|
|
169
|
+
credentials = device_credentials_for(device)
|
|
170
|
+
policy.connection_options(device.vendor).merge(credentials) if credentials
|
|
171
|
+
end
|
|
172
|
+
|
|
173
|
+
def device_credentials_for(device)
|
|
92
174
|
prefix = "NET_CONNECTOR_#{device.vendor.to_s.upcase}_"
|
|
93
|
-
username =
|
|
175
|
+
username = secret_value(prefix + "USERNAME") || secret_value("NET_CONNECTOR_DEVICE_USERNAME")
|
|
94
176
|
return nil unless username
|
|
95
177
|
|
|
96
|
-
{
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
enable_password: value(prefix + "ENABLE_PASSWORD") || value("NET_CONNECTOR_ENABLE_PASSWORD"),
|
|
100
|
-
protocol: (value(prefix + "PROTOCOL") || value("NET_CONNECTOR_PROTOCOL") || "ssh").to_sym,
|
|
101
|
-
log_level: (value("NET_CONNECTOR_LOG_LEVEL") || "info").to_sym,
|
|
102
|
-
known_hosts: value("NET_CONNECTOR_KNOWN_HOSTS"),
|
|
103
|
-
host_key_policy: (value("NET_CONNECTOR_HOST_KEY_POLICY") || "strict").to_sym
|
|
104
|
-
}
|
|
178
|
+
{ username: username,
|
|
179
|
+
password: secret_value(prefix + "PASSWORD") || secret_value("NET_CONNECTOR_DEVICE_PASSWORD"),
|
|
180
|
+
enable_password: secret_value(prefix + "ENABLE_PASSWORD") || secret_value("NET_CONNECTOR_ENABLE_PASSWORD") }
|
|
105
181
|
end
|
|
106
182
|
|
|
107
|
-
# 读取并展开设备日志目录。
|
|
108
183
|
def log_directory
|
|
109
184
|
path = value("NET_CONNECTOR_LOG_DIRECTORY")
|
|
110
185
|
File.expand_path(path) if path
|
|
111
186
|
end
|
|
112
187
|
|
|
113
|
-
# 仅展示非敏感配置,不输出设备凭据或 API 密钥。
|
|
114
|
-
# 生成不包含凭据的可展示配置。
|
|
115
188
|
def public_config
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
189
|
+
snapshot(mode: :show_config).send(:config_hash)
|
|
190
|
+
end
|
|
191
|
+
|
|
192
|
+
def inspect = "#<#{self.class}>"
|
|
193
|
+
|
|
194
|
+
private
|
|
195
|
+
|
|
196
|
+
def config_hash
|
|
124
197
|
{
|
|
125
|
-
netdisco:
|
|
126
|
-
backup: { directory: backup_directory, concurrency: concurrency,
|
|
127
|
-
limit_per_vendor: limit_per_vendor },
|
|
198
|
+
netdisco: client_options,
|
|
199
|
+
backup: { directory: backup_directory, concurrency: concurrency, limit_per_vendor: limit_per_vendor },
|
|
128
200
|
inventory: { include_hosts: list("NET_CONNECTOR_INCLUDE_HOSTS"),
|
|
129
201
|
exclude_hosts: list("NET_CONNECTOR_EXCLUDE_HOSTS"),
|
|
130
202
|
include_vendors: list("NET_CONNECTOR_INCLUDE_VENDORS"),
|
|
131
203
|
vendor_overrides: json("NET_CONNECTOR_VENDOR_OVERRIDES", {}),
|
|
132
204
|
host_overrides: json("NET_CONNECTOR_HOST_OVERRIDES", {}),
|
|
133
205
|
device_rules: json("NET_CONNECTOR_DEVICE_RULES", []) },
|
|
134
|
-
ssh: {
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
206
|
+
ssh: connection_options.transform_values { |item| item.is_a?(Symbol) ? item.to_s : item }.merge(
|
|
207
|
+
log_directory: log_directory,
|
|
208
|
+
vendor_protocols: Net::Connector.vendors.filter_map do |vendor|
|
|
209
|
+
protocol = value("NET_CONNECTOR_#{vendor.to_s.upcase}_PROTOCOL")
|
|
210
|
+
[vendor, protocol] if protocol
|
|
211
|
+
end.to_h
|
|
212
|
+
),
|
|
213
|
+
tftp: { server: tftp_server, vrfs: tftp_vrfs.transform_keys(&:to_s), source_files: tftp_source_files }
|
|
140
214
|
}
|
|
141
|
-
rescue URI::InvalidURIError
|
|
142
|
-
raise ArgumentError, "NETDISCO_URL is invalid"
|
|
143
215
|
end
|
|
144
216
|
|
|
145
|
-
|
|
217
|
+
def non_secret_keys
|
|
218
|
+
keys = ConfigFile::FIELDS.values.flat_map { |section| section.values.map(&:first) }
|
|
219
|
+
keys + Net::Connector.vendors.map { |vendor| "NET_CONNECTOR_#{vendor.to_s.upcase}_PROTOCOL" }
|
|
220
|
+
end
|
|
221
|
+
|
|
222
|
+
def raw(key, default = nil)
|
|
223
|
+
return @policy.send(:raw, key, default) if @policy
|
|
224
|
+
return @overrides.fetch(key) if @overrides.key?(key)
|
|
225
|
+
return @env.fetch(key) if @env.key?(key)
|
|
226
|
+
|
|
227
|
+
@defaults.fetch(key, default)
|
|
228
|
+
end
|
|
229
|
+
|
|
230
|
+
def configured?(key) = !raw(key, UNSET).equal?(UNSET)
|
|
146
231
|
|
|
147
|
-
# 解析指定环境变量中的 JSON 配置。
|
|
148
232
|
def json(key, default)
|
|
149
|
-
|
|
150
|
-
|
|
233
|
+
text = value(key)
|
|
234
|
+
text ? JSON.parse(text) : default
|
|
151
235
|
rescue JSON::ParserError
|
|
152
|
-
raise ArgumentError, "#{key} must be valid JSON"
|
|
236
|
+
raise ArgumentError, "#{key} must be valid JSON", cause: nil
|
|
153
237
|
end
|
|
154
238
|
|
|
155
|
-
|
|
156
|
-
def list(key) = @env.fetch(key, "").split(",").map(&:strip).reject(&:empty?)
|
|
239
|
+
def list(key) = (value(key) || "").split(",").map(&:strip).reject(&:empty?)
|
|
157
240
|
|
|
158
|
-
# 读取正整数配置并统一错误信息。
|
|
159
241
|
def integer(key, default)
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
raise ArgumentError, "#{key} must be a positive integer" unless value.positive?
|
|
242
|
+
result = Integer(raw(key, default).to_s, 10)
|
|
243
|
+
raise ArgumentError unless result.positive?
|
|
163
244
|
|
|
164
|
-
|
|
245
|
+
result
|
|
165
246
|
rescue ArgumentError, TypeError
|
|
166
|
-
raise ArgumentError, "#{key} must be a positive integer"
|
|
247
|
+
raise ArgumentError, "#{key} must be a positive integer", cause: nil
|
|
248
|
+
end
|
|
249
|
+
|
|
250
|
+
def optional_integer(key)
|
|
251
|
+
integer(key, nil) unless raw(key).nil?
|
|
252
|
+
end
|
|
253
|
+
|
|
254
|
+
def number(key, default)
|
|
255
|
+
input = raw(key, default)
|
|
256
|
+
result = input.is_a?(String) ? Float(input) : input
|
|
257
|
+
unless result.is_a?(Numeric) && result.real? && result.finite? && result.positive? && result.to_f.finite?
|
|
258
|
+
raise ArgumentError
|
|
259
|
+
end
|
|
260
|
+
result
|
|
261
|
+
rescue ArgumentError, TypeError
|
|
262
|
+
raise ArgumentError, "#{key} must be a positive finite number", cause: nil
|
|
263
|
+
end
|
|
264
|
+
|
|
265
|
+
def boolean(key, default)
|
|
266
|
+
result = raw(key, default)
|
|
267
|
+
return true if result == true || result == "true"
|
|
268
|
+
return false if result == false || result == "false"
|
|
269
|
+
|
|
270
|
+
raise ArgumentError, "#{key} must be true or false"
|
|
167
271
|
end
|
|
168
272
|
|
|
169
|
-
# 读取非空字符串配置。
|
|
170
273
|
def value(key)
|
|
171
|
-
result =
|
|
274
|
+
result = raw(key)
|
|
275
|
+
raise ArgumentError, "#{key} must be a String" unless result.nil? || result.is_a?(String)
|
|
276
|
+
|
|
172
277
|
result if present?(result)
|
|
173
278
|
end
|
|
174
279
|
|
|
175
|
-
|
|
176
|
-
|
|
280
|
+
def secret_value(key)
|
|
281
|
+
result = @env[key]
|
|
282
|
+
raise ArgumentError, "#{key} must be a String" unless result.nil? || result.is_a?(String)
|
|
283
|
+
|
|
284
|
+
result.dup.freeze if present?(result)
|
|
285
|
+
end
|
|
177
286
|
|
|
178
|
-
#
|
|
287
|
+
def required_secret(key) = secret_value(key) || (raise ArgumentError, "#{key} is required")
|
|
179
288
|
def present?(value) = value.is_a?(String) && !value.strip.empty?
|
|
180
289
|
end
|
|
181
290
|
end
|
|
@@ -31,25 +31,10 @@ module Net
|
|
|
31
31
|
Thread.current.report_on_exception = false
|
|
32
32
|
while (task = queue.pop)
|
|
33
33
|
index, device = task
|
|
34
|
-
|
|
35
|
-
begin
|
|
36
|
-
on_start&.call(device)
|
|
37
|
-
rescue StandardError => error
|
|
38
|
-
callback_errors << { host: device.host, error_type: error.class.name }.freeze
|
|
39
|
-
end
|
|
40
|
-
result = begin
|
|
41
|
-
yield device
|
|
42
|
-
rescue StandardError => error
|
|
43
|
-
on_error.call(device, error)
|
|
44
|
-
end
|
|
45
|
-
result = result.with(started_at: started_at, finished_at: Time.now.utc)
|
|
34
|
+
result = run_task(device, on_error, on_start, callback_errors) { yield device }
|
|
46
35
|
outcomes[index] = result
|
|
47
36
|
Array(on_result).each do |callback|
|
|
48
|
-
|
|
49
|
-
callback.call(result)
|
|
50
|
-
rescue StandardError => error
|
|
51
|
-
callback_errors << { host: device.host, error_type: error.class.name }.freeze
|
|
52
|
-
end
|
|
37
|
+
notify(callback, result, host: device.host, errors: callback_errors)
|
|
53
38
|
end
|
|
54
39
|
end
|
|
55
40
|
ensure
|
|
@@ -76,6 +61,30 @@ module Net
|
|
|
76
61
|
end
|
|
77
62
|
Array.new(callback_errors.size) { callback_errors.pop }.freeze
|
|
78
63
|
end
|
|
64
|
+
|
|
65
|
+
private
|
|
66
|
+
|
|
67
|
+
# 普通设备故障转换为结果,中断仍交给批次统一停止其余线程。
|
|
68
|
+
def run_task(device, on_error, on_start, callback_errors)
|
|
69
|
+
started_at = Time.now.utc
|
|
70
|
+
started = monotonic
|
|
71
|
+
notify(on_start, device, host: device.host, errors: callback_errors) unless on_start.nil?
|
|
72
|
+
result = begin
|
|
73
|
+
yield
|
|
74
|
+
rescue StandardError => error
|
|
75
|
+
on_error.call(device, error)
|
|
76
|
+
end
|
|
77
|
+
result.with(started_at: started_at, finished_at: Time.now.utc, duration_ms: ((monotonic - started) * 1000).round)
|
|
78
|
+
end
|
|
79
|
+
|
|
80
|
+
def monotonic = Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
|
81
|
+
|
|
82
|
+
# 回调故障不覆盖设备结果,也不把异常消息中的凭据写入报告。
|
|
83
|
+
def notify(callback, value, host:, errors:)
|
|
84
|
+
callback.call(value)
|
|
85
|
+
rescue StandardError => error
|
|
86
|
+
errors << { host: host, error_type: error.class.name }.freeze
|
|
87
|
+
end
|
|
79
88
|
end
|
|
80
89
|
end
|
|
81
90
|
end
|
|
@@ -0,0 +1,116 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "digest"
|
|
4
|
+
require_relative "../engine/errors"
|
|
5
|
+
|
|
6
|
+
module Net
|
|
7
|
+
module Connector
|
|
8
|
+
class BackupBusy < Error; end
|
|
9
|
+
|
|
10
|
+
module Operations
|
|
11
|
+
# 路径锁在设备采集之前取得;锁文件长期保留,避免等待者落在已经删除的 inode 上。
|
|
12
|
+
class BackupLock
|
|
13
|
+
STORAGE_KEY = :net_connector_backup_delegation
|
|
14
|
+
private_constant :STORAGE_KEY
|
|
15
|
+
|
|
16
|
+
attr_reader :path
|
|
17
|
+
|
|
18
|
+
def self.lock_path(path)
|
|
19
|
+
destination = File.expand_path(path)
|
|
20
|
+
directory = File.realpath(File.dirname(destination))
|
|
21
|
+
# 锁名先折叠 Unicode 与大小写,避免大小写不敏感文件系统上的别名绕过互斥。
|
|
22
|
+
# 在区分大小写的文件系统上可能保守地合并两个锁,但不会改写目标文件名。
|
|
23
|
+
filename = File.basename(destination).b.force_encoding(Encoding::UTF_8)
|
|
24
|
+
filename = filename.valid_encoding? ? filename.unicode_normalize(:nfc).downcase(:fold).unicode_normalize(:nfc) : filename.b.downcase
|
|
25
|
+
File.join(directory, ".net-connector-#{Digest::SHA256.hexdigest(filename)}.lock")
|
|
26
|
+
end
|
|
27
|
+
|
|
28
|
+
def self.synchronize(path, **options, &block) = new(path, **options).synchronize(&block)
|
|
29
|
+
|
|
30
|
+
def initialize(path, timeout: 0, host: nil)
|
|
31
|
+
raise ArgumentError, "path must be a nonempty String" unless path.is_a?(String) && !path.empty?
|
|
32
|
+
unless timeout.is_a?(Numeric) && timeout.real? && timeout.finite? && timeout >= 0 && timeout.to_f.finite?
|
|
33
|
+
raise ArgumentError, "lock_timeout must be a nonnegative finite number"
|
|
34
|
+
end
|
|
35
|
+
|
|
36
|
+
@path = File.expand_path(path).freeze
|
|
37
|
+
@lock_path = self.class.lock_path(@path).freeze
|
|
38
|
+
@timeout, @host = timeout.to_f, host
|
|
39
|
+
end
|
|
40
|
+
|
|
41
|
+
def synchronize
|
|
42
|
+
# Fleet 只向下层备份借用一次;消费后清除,采集回调中的递归备份仍须竞争锁。
|
|
43
|
+
# Thread#[] 是 Fiber 局部存储;fork 后不能消费父进程的借用记录。
|
|
44
|
+
if Thread.current[STORAGE_KEY] == [@lock_path, Process.pid]
|
|
45
|
+
Thread.current[STORAGE_KEY] = nil
|
|
46
|
+
return yield self
|
|
47
|
+
end
|
|
48
|
+
|
|
49
|
+
file = nil
|
|
50
|
+
begin
|
|
51
|
+
file = open_lock
|
|
52
|
+
verify_lock!(file)
|
|
53
|
+
acquire(file)
|
|
54
|
+
verify_lock!(file)
|
|
55
|
+
@owner = [Process.pid, Fiber.current]
|
|
56
|
+
yield self
|
|
57
|
+
ensure
|
|
58
|
+
@owner = nil
|
|
59
|
+
file&.close
|
|
60
|
+
end
|
|
61
|
+
end
|
|
62
|
+
|
|
63
|
+
# 仅持有实际 flock 的同一 Fiber 可授权一次内层采集,避免把路径锁变成任意可重入锁。
|
|
64
|
+
def delegate
|
|
65
|
+
raise ThreadError, "backup path delegation requires lock ownership" unless @owner == [Process.pid, Fiber.current]
|
|
66
|
+
|
|
67
|
+
previous = Thread.current[STORAGE_KEY]
|
|
68
|
+
begin
|
|
69
|
+
Thread.current[STORAGE_KEY] = [@lock_path, Process.pid]
|
|
70
|
+
yield
|
|
71
|
+
ensure
|
|
72
|
+
Thread.current[STORAGE_KEY] = previous
|
|
73
|
+
end
|
|
74
|
+
end
|
|
75
|
+
|
|
76
|
+
private
|
|
77
|
+
|
|
78
|
+
def open_lock
|
|
79
|
+
File.open(@lock_path, File::RDWR | File::CREAT | File::NOFOLLOW | File::NONBLOCK, 0o600)
|
|
80
|
+
rescue Errno::ELOOP, Errno::EISDIR, Errno::ENXIO
|
|
81
|
+
raise ArgumentError, "backup lock must be a private regular file", cause: nil
|
|
82
|
+
end
|
|
83
|
+
|
|
84
|
+
def verify_lock!(file)
|
|
85
|
+
stat = file.stat
|
|
86
|
+
unless stat.file? && stat.uid == Process.euid && stat.nlink == 1 && (stat.mode & 0o7777) == 0o600
|
|
87
|
+
raise ArgumentError, "backup lock must be an owned regular file with mode 0600 and one link"
|
|
88
|
+
end
|
|
89
|
+
current = File.lstat(@lock_path)
|
|
90
|
+
unless current.file? && current.dev == stat.dev && current.ino == stat.ino
|
|
91
|
+
raise ArgumentError, "backup lock path changed while acquiring ownership"
|
|
92
|
+
end
|
|
93
|
+
rescue Errno::ENOENT
|
|
94
|
+
raise ArgumentError, "backup lock path changed while acquiring ownership", cause: nil
|
|
95
|
+
end
|
|
96
|
+
|
|
97
|
+
def acquire(file)
|
|
98
|
+
deadline = monotonic + @timeout
|
|
99
|
+
loop do
|
|
100
|
+
acquired = file.flock(File::LOCK_EX | File::LOCK_NB)
|
|
101
|
+
return if acquired && (@timeout.zero? || monotonic < deadline)
|
|
102
|
+
|
|
103
|
+
remaining = deadline - monotonic
|
|
104
|
+
if remaining <= 0
|
|
105
|
+
raise BackupBusy.new("backup path already belongs to another operation", host: @host, phase: :backup), cause: nil
|
|
106
|
+
end
|
|
107
|
+
wait([remaining, 0.05].min)
|
|
108
|
+
end
|
|
109
|
+
end
|
|
110
|
+
|
|
111
|
+
def monotonic = Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
|
112
|
+
def wait(seconds) = sleep(seconds)
|
|
113
|
+
end
|
|
114
|
+
end
|
|
115
|
+
end
|
|
116
|
+
end
|
|
@@ -2,6 +2,8 @@
|
|
|
2
2
|
|
|
3
3
|
require "digest"
|
|
4
4
|
require_relative "private_file"
|
|
5
|
+
require_relative "backup_lock"
|
|
6
|
+
require_relative "safe_file"
|
|
5
7
|
|
|
6
8
|
module Net
|
|
7
9
|
module Connector
|
|
@@ -15,19 +17,47 @@ module Net
|
|
|
15
17
|
def changed? = [:created, :changed].include?(change)
|
|
16
18
|
end
|
|
17
19
|
|
|
20
|
+
# 只携带库自身的提交回执和配置元数据,不携带正文或原始文件系统异常。
|
|
21
|
+
class BackupPersistenceError < Error
|
|
22
|
+
attr_reader :backup, :receipt, :underlying_type
|
|
23
|
+
|
|
24
|
+
def initialize(backup:, write_error:, host: nil)
|
|
25
|
+
unless backup.instance_of?(Backup) && Operations::PrivateFile.receipt_error?(write_error) &&
|
|
26
|
+
write_error.receipt.committed? && write_error.receipt.path == backup.path
|
|
27
|
+
raise ArgumentError, "backup persistence error requires a matching committed backup receipt"
|
|
28
|
+
end
|
|
29
|
+
|
|
30
|
+
@backup, @receipt = backup, write_error.receipt
|
|
31
|
+
@underlying_type = write_error.underlying_type
|
|
32
|
+
code = if write_error.instance_of?(Operations::PrivateFile::DirectorySyncUnsupported)
|
|
33
|
+
:backup_durability_unsupported
|
|
34
|
+
elsif receipt.durable?
|
|
35
|
+
:backup_finalize_failed
|
|
36
|
+
else
|
|
37
|
+
:backup_persistence_unconfirmed
|
|
38
|
+
end
|
|
39
|
+
super(write_error.message, code: code, host: host, phase: :backup)
|
|
40
|
+
end
|
|
41
|
+
end
|
|
42
|
+
|
|
18
43
|
module Operations
|
|
19
44
|
class LocalBackup
|
|
20
45
|
# 保存执行配置采集的设备对象。
|
|
21
46
|
def initialize(device) = @device = device
|
|
22
47
|
|
|
23
48
|
# 采集配置、识别变更并原子写入私有文件。
|
|
24
|
-
def call(path:)
|
|
25
|
-
|
|
49
|
+
def call(path:, lock_timeout: 0)
|
|
50
|
+
host = @device.host if @device.respond_to?(:host)
|
|
51
|
+
BackupLock.synchronize(path, timeout: lock_timeout, host: host) { |lock| collect_and_write(lock.path) }
|
|
52
|
+
end
|
|
53
|
+
|
|
54
|
+
private
|
|
26
55
|
|
|
56
|
+
def collect_and_write(destination)
|
|
27
57
|
contents = @device.running_config.value!
|
|
28
|
-
destination = File.expand_path(path)
|
|
29
58
|
digest = Digest::SHA256.hexdigest(contents)
|
|
30
|
-
|
|
59
|
+
previous = SafeFile.fingerprint(destination, missing: true, replace_symlink: true)
|
|
60
|
+
previous_digest = previous&.sha256
|
|
31
61
|
change = if previous_digest.nil?
|
|
32
62
|
:created
|
|
33
63
|
elsif previous_digest == digest
|
|
@@ -36,12 +66,22 @@ module Net
|
|
|
36
66
|
:changed
|
|
37
67
|
end
|
|
38
68
|
|
|
39
|
-
|
|
40
|
-
|
|
69
|
+
backup = Backup.new(path: destination, bytes: contents.bytesize,
|
|
70
|
+
sha256: digest, collected_at: Time.now.utc, change: change,
|
|
71
|
+
previous_sha256: previous_digest)
|
|
72
|
+
if change != :unchanged || previous.mode != 0o600 || !SafeFile.same_entry?(destination, previous)
|
|
73
|
+
write_backup(backup, contents)
|
|
41
74
|
end
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
75
|
+
backup
|
|
76
|
+
end
|
|
77
|
+
|
|
78
|
+
def write_backup(backup, contents)
|
|
79
|
+
PrivateFile.write_receipt(backup.path, contents)
|
|
80
|
+
rescue PrivateFile::WriteError => error
|
|
81
|
+
raise unless PrivateFile.receipt_error?(error) && error.receipt.committed?
|
|
82
|
+
|
|
83
|
+
host = @device.host if @device.respond_to?(:host)
|
|
84
|
+
raise BackupPersistenceError.new(backup: backup, write_error: error, host: host), cause: nil
|
|
45
85
|
end
|
|
46
86
|
end
|
|
47
87
|
end
|