net-connector 0.6.0 → 0.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +52 -0
- data/CONTRIBUTING.md +20 -0
- data/README.md +112 -50
- data/docs/VERIFICATION.md +42 -1
- data/docs/architecture.md +15 -2
- data/examples/backup.yml +40 -0
- data/lib/net/connector/device/base.rb +30 -38
- data/lib/net/connector/device/profile/builder.rb +8 -8
- data/lib/net/connector/device/profile.rb +2 -2
- data/lib/net/connector/device/running_config.rb +2 -2
- data/lib/net/connector/device/tftp/receipt.rb +12 -3
- data/lib/net/connector/device/tftp.rb +7 -7
- data/lib/net/connector/device/topology.rb +4 -4
- data/lib/net/connector/engine/command.rb +2 -2
- data/lib/net/connector/engine/configuration.rb +5 -2
- data/lib/net/connector/engine/execution.rb +25 -14
- data/lib/net/connector/engine/known_hosts.rb +112 -0
- data/lib/net/connector/engine/log.rb +10 -7
- data/lib/net/connector/engine/session.rb +117 -128
- data/lib/net/connector/engine/transport.rb +32 -7
- data/lib/net/connector/netdisco/backup_run.rb +132 -0
- data/lib/net/connector/netdisco/batch.rb +1 -1
- data/lib/net/connector/netdisco/cli/options.rb +139 -0
- data/lib/net/connector/netdisco/cli.rb +13 -26
- data/lib/net/connector/netdisco/client.rb +6 -6
- data/lib/net/connector/netdisco/config_file.rb +20 -19
- data/lib/net/connector/netdisco/connection.rb +38 -0
- data/lib/net/connector/netdisco/database_client.rb +2 -2
- data/lib/net/connector/netdisco/device.rb +11 -1
- data/lib/net/connector/netdisco/fleet.rb +68 -47
- data/lib/net/connector/netdisco/inventory_budget.rb +4 -4
- data/lib/net/connector/netdisco/plan.rb +14 -9
- data/lib/net/connector/netdisco/planner.rb +18 -14
- data/lib/net/connector/netdisco/progress.rb +253 -0
- data/lib/net/connector/netdisco/report/files.rb +41 -0
- data/lib/net/connector/netdisco/report/text.rb +45 -0
- data/lib/net/connector/netdisco/report.rb +53 -31
- data/lib/net/connector/netdisco/result_store.rb +3 -1
- data/lib/net/connector/netdisco/rules.rb +10 -2
- data/lib/net/connector/netdisco/settings.rb +118 -68
- data/lib/net/connector/netdisco/tftp_archive.rb +175 -0
- data/lib/net/connector/netdisco/tftp_history.rb +16 -0
- data/lib/net/connector/netdisco/tftp_verification.rb +58 -0
- data/lib/net/connector/netdisco/worker.rb +8 -3
- data/lib/net/connector/netdisco.rb +10 -0
- data/lib/net/connector/storage/backup_lock.rb +1 -1
- data/lib/net/connector/storage/batch_directory.rb +29 -0
- data/lib/net/connector/storage.rb +1 -0
- data/lib/net/connector/vendor/h3c.rb +2 -1
- data/lib/net/connector/vendor/hillstone/tftp_backup.rb +3 -3
- data/lib/net/connector/vendor/palo_alto/tftp_backup.rb +3 -3
- data/lib/net/connector/vendor/radware/running_config.rb +19 -0
- data/lib/net/connector/vendor/radware.rb +4 -3
- data/lib/net/connector/version.rb +1 -1
- data/lib/net/connector.rb +2 -2
- metadata +42 -111
- /data/examples/{netdisco_database.yml → inventory_sql.yml} +0 -0
|
@@ -10,6 +10,8 @@ module Net
|
|
|
10
10
|
DEFAULT_BACKUP_DIRECTORY = "backups"
|
|
11
11
|
DEFAULT_CONCURRENCY = 4
|
|
12
12
|
MAX_CONCURRENCY = 50
|
|
13
|
+
ENVIRONMENT_KEYS = %w[NETDISCO_URL NC_BACKUP_DIRECTORY NC_CONCURRENCY NC_SAMPLE_PER_VENDOR
|
|
14
|
+
NC_PROTOCOL NC_KNOWN_HOSTS NC_HOST_KEY_POLICY NC_LOG_DIRECTORY NC_LOG_LEVEL TFTP_HOST].freeze
|
|
13
15
|
DEFAULT_PAGE_SIZE = Client::DEFAULTS.fetch(:page_size)
|
|
14
16
|
UNSET = Object.new.freeze
|
|
15
17
|
private_constant :UNSET
|
|
@@ -18,17 +20,29 @@ module Net
|
|
|
18
20
|
new(env: env, defaults: ConfigFile.load(path))
|
|
19
21
|
end
|
|
20
22
|
|
|
23
|
+
def self.from_env(env: ENV)
|
|
24
|
+
path = env["NC_CONFIG"]
|
|
25
|
+
path && !path.empty? ? from_file(path, env: env) : new(env: env)
|
|
26
|
+
end
|
|
27
|
+
|
|
21
28
|
def initialize(env: ENV, defaults: {}, overrides: {}, policy: nil, fixed: false)
|
|
22
29
|
@env, @defaults, @overrides = env, defaults, overrides
|
|
23
30
|
@policy, @fixed = policy, fixed
|
|
24
31
|
end
|
|
25
32
|
|
|
33
|
+
# 调用入口补充运行目录等非敏感覆盖,凭据仍从同一来源读取。
|
|
34
|
+
def with_overrides(values)
|
|
35
|
+
self.class.new(env: @env, defaults: @defaults, overrides: @overrides.merge(values))
|
|
36
|
+
end
|
|
37
|
+
|
|
26
38
|
# 快照自身不保留 ENV、凭据 resolver 或原 Settings 的引用,不能携带秘密进入摘要。
|
|
27
39
|
def snapshot(mode: :backup)
|
|
28
40
|
return @policy.validate!(mode: mode) if @policy
|
|
29
41
|
return validate!(mode: mode) if @fixed
|
|
30
42
|
|
|
31
|
-
|
|
43
|
+
validate_environment! unless mode == :export
|
|
44
|
+
|
|
45
|
+
keys = mode == :export ? ["NC_BACKUP_DIRECTORY"] : non_secret_keys
|
|
32
46
|
values = keys.each_with_object({}) do |key, result|
|
|
33
47
|
item = raw(key, UNSET)
|
|
34
48
|
result[key.freeze] = item.is_a?(String) ? item.dup.freeze : item unless item.equal?(UNSET)
|
|
@@ -61,18 +75,25 @@ module Net
|
|
|
61
75
|
self
|
|
62
76
|
end
|
|
63
77
|
|
|
64
|
-
def client(policy: snapshot(mode: :inventory))
|
|
78
|
+
def client(policy: snapshot(mode: :inventory), credentials: nil)
|
|
79
|
+
if credentials && (!credentials.is_a?(Hash) || credentials.keys.sort != %i[password username] ||
|
|
80
|
+
!credentials.values.all? { |value| present?(value) })
|
|
81
|
+
raise ArgumentError, "inventory credentials require nonempty username and password"
|
|
82
|
+
end
|
|
65
83
|
options = policy.client_options
|
|
66
84
|
if policy.inventory_source == :postgres
|
|
67
85
|
connection = DatabaseClient::CONNECTION_ENV.each_with_object({}) do |(key, name), values|
|
|
68
86
|
item = secret_value(name)
|
|
69
87
|
values[key] = item if item
|
|
70
88
|
end
|
|
89
|
+
connection.merge!(user: credentials[:username], password: credentials[:password]) if credentials
|
|
71
90
|
return DatabaseClient.new(connection_options: connection, **options)
|
|
72
91
|
end
|
|
73
92
|
raise ArgumentError, "NETDISCO_URL is required" unless options.fetch(:url)
|
|
74
93
|
|
|
75
|
-
authentication = if
|
|
94
|
+
authentication = if credentials
|
|
95
|
+
credentials
|
|
96
|
+
elsif (token = secret_value("NETDISCO_API_KEY"))
|
|
76
97
|
{ api_key: token }
|
|
77
98
|
else
|
|
78
99
|
{ username: required_secret("NETDISCO_USERNAME"), password: required_secret("NETDISCO_PASSWORD") }
|
|
@@ -90,54 +111,36 @@ module Net
|
|
|
90
111
|
# 原生客户端和 --show-config 共享预算校验;只验证所选来源的查询设置。
|
|
91
112
|
def client_options
|
|
92
113
|
defaults = inventory_source == :postgres ? DatabaseClient::DEFAULTS : Client::DEFAULTS
|
|
93
|
-
options = defaults
|
|
94
|
-
|
|
95
|
-
parsed = case name
|
|
96
|
-
when :inventory_timeout then number(key, default)
|
|
97
|
-
when :allow_insecure_http then boolean(key, default)
|
|
98
|
-
else integer(key, default)
|
|
99
|
-
end
|
|
100
|
-
[name, parsed]
|
|
101
|
-
end
|
|
102
|
-
if inventory_source == :postgres
|
|
103
|
-
query = value("NETDISCO_QUERY") || (raise ArgumentError, "NETDISCO_QUERY is required for postgres")
|
|
104
|
-
return DatabaseClient.options(query: query, query_params: json("NETDISCO_QUERY_PARAMS", []), **options)
|
|
105
|
-
end
|
|
106
|
-
if value("NETDISCO_QUERY") || value("NETDISCO_QUERY_PARAMS")
|
|
107
|
-
raise ArgumentError, "SQL query settings require NETDISCO_SOURCE=postgres"
|
|
108
|
-
end
|
|
109
|
-
options = Client.options(**options)
|
|
110
|
-
url = value("NETDISCO_URL")
|
|
111
|
-
Client.validate_url!(url, allow_insecure_http: options.fetch(:allow_insecure_http)) if url
|
|
112
|
-
options.merge(url: url).freeze
|
|
114
|
+
options = client_budget_options(defaults)
|
|
115
|
+
inventory_source == :postgres ? postgres_options(options) : http_options(options)
|
|
113
116
|
end
|
|
114
117
|
|
|
115
118
|
def rules
|
|
116
|
-
Rules.new(include_hosts: list("
|
|
117
|
-
exclude_hosts: list("
|
|
118
|
-
include_vendors: list("
|
|
119
|
-
vendor_overrides: json("
|
|
120
|
-
host_overrides: json("
|
|
121
|
-
mappings: json("
|
|
119
|
+
Rules.new(include_hosts: list("NC_INCLUDE_HOSTS"),
|
|
120
|
+
exclude_hosts: list("NC_EXCLUDE_HOSTS"),
|
|
121
|
+
include_vendors: list("NC_INCLUDE_VENDORS"),
|
|
122
|
+
vendor_overrides: json("NC_VENDOR_OVERRIDES", {}),
|
|
123
|
+
host_overrides: json("NC_HOST_OVERRIDES", {}),
|
|
124
|
+
mappings: json("NC_DEVICE_RULES", []))
|
|
122
125
|
end
|
|
123
126
|
|
|
124
127
|
def backup_directory
|
|
125
|
-
path = raw("
|
|
126
|
-
raise ArgumentError, "
|
|
128
|
+
path = raw("NC_BACKUP_DIRECTORY", DEFAULT_BACKUP_DIRECTORY)
|
|
129
|
+
raise ArgumentError, "NC_BACKUP_DIRECTORY must be nonempty" unless present?(path)
|
|
127
130
|
|
|
128
131
|
File.expand_path(path)
|
|
129
132
|
end
|
|
130
133
|
|
|
131
134
|
def concurrency
|
|
132
|
-
count = integer("
|
|
133
|
-
raise ArgumentError, "
|
|
135
|
+
count = integer("NC_CONCURRENCY", DEFAULT_CONCURRENCY)
|
|
136
|
+
raise ArgumentError, "NC_CONCURRENCY must be at most #{MAX_CONCURRENCY}" if count > MAX_CONCURRENCY
|
|
134
137
|
|
|
135
138
|
count
|
|
136
139
|
end
|
|
137
140
|
|
|
138
141
|
def limit_per_vendor(tftp: false)
|
|
139
|
-
count = if configured?("
|
|
140
|
-
integer("
|
|
142
|
+
count = if configured?("NC_SAMPLE_PER_VENDOR")
|
|
143
|
+
integer("NC_SAMPLE_PER_VENDOR", 5) unless raw("NC_SAMPLE_PER_VENDOR").nil?
|
|
141
144
|
elsif tftp
|
|
142
145
|
5
|
|
143
146
|
end
|
|
@@ -146,17 +149,17 @@ module Net
|
|
|
146
149
|
end
|
|
147
150
|
|
|
148
151
|
def tftp_source_files
|
|
149
|
-
files = { huawei: raw("
|
|
152
|
+
files = { huawei: raw("NC_HUAWEI_TFTP_SOURCE_FILE", "flash:/startup.cfg") }
|
|
150
153
|
%i[h3c h3c_wireless].each do |vendor|
|
|
151
|
-
key = "
|
|
154
|
+
key = "NC_#{vendor.to_s.upcase}_TFTP_SOURCE_FILE"
|
|
152
155
|
files[vendor] = raw(key) if configured?(key)
|
|
153
156
|
end
|
|
154
157
|
files
|
|
155
158
|
end
|
|
156
159
|
|
|
157
160
|
def tftp_vrfs
|
|
158
|
-
mapping = json("
|
|
159
|
-
raise ArgumentError, "
|
|
161
|
+
mapping = json("NC_TFTP_VRFS", {})
|
|
162
|
+
raise ArgumentError, "NC_TFTP_VRFS must be a JSON object" unless mapping.is_a?(Hash)
|
|
160
163
|
|
|
161
164
|
self.class.validate_vrfs!(mapping.transform_keys(&:to_sym))
|
|
162
165
|
end
|
|
@@ -174,75 +177,122 @@ module Net
|
|
|
174
177
|
def tftp_server = value("TFTP_HOST")
|
|
175
178
|
|
|
176
179
|
def connection_options(vendor = nil)
|
|
177
|
-
prefix = "NET_CONNECTOR_#{vendor.to_s.upcase}_"
|
|
178
180
|
configuration = Configuration.new(
|
|
179
|
-
protocol:
|
|
180
|
-
log_level: value("
|
|
181
|
-
host_key_policy: value("
|
|
182
|
-
max_script_output_bytes: optional_integer("
|
|
181
|
+
protocol: vendor_protocols[vendor.to_s] || value("NC_PROTOCOL") || "ssh",
|
|
182
|
+
log_level: value("NC_LOG_LEVEL") || "info", known_hosts: value("NC_KNOWN_HOSTS"),
|
|
183
|
+
host_key_policy: value("NC_HOST_KEY_POLICY") || "strict",
|
|
184
|
+
max_script_output_bytes: optional_integer("NC_MAX_SCRIPT_OUTPUT_BYTES")
|
|
183
185
|
)
|
|
184
186
|
options = %i[protocol log_level known_hosts host_key_policy].to_h { |key| [key, configuration.public_send(key)] }
|
|
185
187
|
options[:max_script_output_bytes] = configuration.max_script_output_bytes if configuration.max_script_output_bytes
|
|
186
188
|
options.freeze
|
|
187
189
|
end
|
|
188
190
|
|
|
191
|
+
def vendor_protocols
|
|
192
|
+
values = json("NC_VENDOR_PROTOCOLS", {})
|
|
193
|
+
unless values.is_a?(Hash) && values.all? { |vendor, protocol| Net::Connector.vendors.map(&:to_s).include?(vendor) && %w[ssh telnet].include?(protocol) }
|
|
194
|
+
raise ArgumentError, "ssh.vendor_protocols must map supported vendors to ssh or telnet"
|
|
195
|
+
end
|
|
196
|
+
values
|
|
197
|
+
end
|
|
198
|
+
|
|
189
199
|
def device_credentials_for(device)
|
|
190
|
-
prefix = "
|
|
191
|
-
username = secret_value(prefix + "USERNAME") || secret_value("
|
|
200
|
+
prefix = "NC_#{device.vendor.to_s.upcase}_"
|
|
201
|
+
username = secret_value(prefix + "USERNAME") || secret_value("NC_DEVICE_USERNAME")
|
|
192
202
|
return nil unless username
|
|
193
203
|
|
|
194
204
|
{ username: username,
|
|
195
|
-
password: secret_value(prefix + "PASSWORD") || secret_value("
|
|
196
|
-
enable_password: secret_value(prefix + "ENABLE_PASSWORD") || secret_value("
|
|
205
|
+
password: secret_value(prefix + "PASSWORD") || secret_value("NC_DEVICE_PASSWORD"),
|
|
206
|
+
enable_password: secret_value(prefix + "ENABLE_PASSWORD") || secret_value("NC_ENABLE_PASSWORD") }
|
|
197
207
|
end
|
|
198
208
|
|
|
199
209
|
def log_directory
|
|
200
|
-
path = value("
|
|
210
|
+
path = value("NC_LOG_DIRECTORY")
|
|
201
211
|
File.expand_path(path) if path
|
|
202
212
|
end
|
|
203
213
|
|
|
204
214
|
def public_config
|
|
205
|
-
snapshot(mode: :show_config).
|
|
215
|
+
snapshot(mode: :show_config).config_hash
|
|
206
216
|
end
|
|
207
217
|
|
|
208
218
|
def inspect = "#<#{self.class}>"
|
|
209
219
|
|
|
210
|
-
|
|
220
|
+
protected
|
|
211
221
|
|
|
212
222
|
def config_hash
|
|
213
223
|
{
|
|
214
224
|
netdisco: client_options.merge(source: inventory_source),
|
|
215
225
|
backup: { directory: backup_directory, concurrency: concurrency, limit_per_vendor: limit_per_vendor },
|
|
216
|
-
inventory: { include_hosts: list("
|
|
217
|
-
exclude_hosts: list("
|
|
218
|
-
include_vendors: list("
|
|
219
|
-
vendor_overrides: json("
|
|
220
|
-
host_overrides: json("
|
|
221
|
-
device_rules: json("
|
|
226
|
+
inventory: { include_hosts: list("NC_INCLUDE_HOSTS"),
|
|
227
|
+
exclude_hosts: list("NC_EXCLUDE_HOSTS"),
|
|
228
|
+
include_vendors: list("NC_INCLUDE_VENDORS"),
|
|
229
|
+
vendor_overrides: json("NC_VENDOR_OVERRIDES", {}),
|
|
230
|
+
host_overrides: json("NC_HOST_OVERRIDES", {}),
|
|
231
|
+
device_rules: json("NC_DEVICE_RULES", []) },
|
|
222
232
|
ssh: connection_options.transform_values { |item| item.is_a?(Symbol) ? item.to_s : item }.merge(
|
|
223
233
|
log_directory: log_directory,
|
|
224
|
-
vendor_protocols:
|
|
225
|
-
protocol = value("NET_CONNECTOR_#{vendor.to_s.upcase}_PROTOCOL")
|
|
226
|
-
[vendor, protocol] if protocol
|
|
227
|
-
end.to_h
|
|
234
|
+
vendor_protocols: vendor_protocols
|
|
228
235
|
),
|
|
229
236
|
tftp: { server: tftp_server, vrfs: tftp_vrfs.transform_keys(&:to_s), source_files: tftp_source_files }
|
|
230
237
|
}
|
|
231
238
|
end
|
|
232
239
|
|
|
233
|
-
def non_secret_keys
|
|
234
|
-
keys = ConfigFile::FIELDS.values.flat_map { |section| section.values.map(&:first) }
|
|
235
|
-
keys + Net::Connector.vendors.map { |vendor| "NET_CONNECTOR_#{vendor.to_s.upcase}_PROTOCOL" }
|
|
236
|
-
end
|
|
237
|
-
|
|
238
240
|
def raw(key, default = nil)
|
|
239
|
-
return @policy.
|
|
241
|
+
return @policy.raw(key, default) if @policy
|
|
240
242
|
return @overrides.fetch(key) if @overrides.key?(key)
|
|
241
|
-
|
|
243
|
+
if @env.key?(key)
|
|
244
|
+
unless @fixed || ENVIRONMENT_KEYS.include?(key)
|
|
245
|
+
raise ArgumentError, "#{key} is no longer an environment setting; use YAML via NC_CONFIG or CLI options"
|
|
246
|
+
end
|
|
247
|
+
return @env.fetch(key)
|
|
248
|
+
end
|
|
242
249
|
|
|
243
250
|
@defaults.fetch(key, default)
|
|
244
251
|
end
|
|
245
252
|
|
|
253
|
+
private
|
|
254
|
+
|
|
255
|
+
def client_budget_options(defaults)
|
|
256
|
+
defaults.to_h do |name, default|
|
|
257
|
+
key = "NETDISCO_#{name.to_s.upcase}"
|
|
258
|
+
parsed = case name
|
|
259
|
+
when :inventory_timeout then number(key, default)
|
|
260
|
+
when :allow_insecure_http then boolean(key, default)
|
|
261
|
+
else integer(key, default)
|
|
262
|
+
end
|
|
263
|
+
[name, parsed]
|
|
264
|
+
end
|
|
265
|
+
end
|
|
266
|
+
|
|
267
|
+
def postgres_options(options)
|
|
268
|
+
query = value("NETDISCO_QUERY") || (raise ArgumentError, "NETDISCO_QUERY is required for postgres")
|
|
269
|
+
DatabaseClient.options(query: query, query_params: json("NETDISCO_QUERY_PARAMS", []), **options)
|
|
270
|
+
end
|
|
271
|
+
|
|
272
|
+
def http_options(options)
|
|
273
|
+
if value("NETDISCO_QUERY") || value("NETDISCO_QUERY_PARAMS")
|
|
274
|
+
raise ArgumentError, "SQL query settings require NETDISCO_SOURCE=postgres"
|
|
275
|
+
end
|
|
276
|
+
options = Client.options(**options)
|
|
277
|
+
url = value("NETDISCO_URL")
|
|
278
|
+
Client.validate_url!(url, allow_insecure_http: options.fetch(:allow_insecure_http)) if url
|
|
279
|
+
options.merge(url: url).freeze
|
|
280
|
+
end
|
|
281
|
+
|
|
282
|
+
def non_secret_keys
|
|
283
|
+
ConfigFile::FIELDS.values.flat_map { |section| section.values.map(&:first) }
|
|
284
|
+
end
|
|
285
|
+
|
|
286
|
+
def validate_environment!
|
|
287
|
+
legacy = @env.keys.find { |key| key.start_with?("NET_CONNECTOR_") }
|
|
288
|
+
raise ArgumentError, "#{legacy} has been renamed to #{legacy.sub("NET_CONNECTOR_", "NC_")}" if legacy
|
|
289
|
+
|
|
290
|
+
removed = non_secret_keys - ENVIRONMENT_KEYS
|
|
291
|
+
removed += Net::Connector.vendors.map { |vendor| "NC_#{vendor.to_s.upcase}_PROTOCOL" }
|
|
292
|
+
key = removed.find { |name| @env.key?(name) }
|
|
293
|
+
raise ArgumentError, "#{key} is no longer an environment setting; use YAML via NC_CONFIG or CLI options" if key
|
|
294
|
+
end
|
|
295
|
+
|
|
246
296
|
def configured?(key) = !raw(key, UNSET).equal?(UNSET)
|
|
247
297
|
|
|
248
298
|
def json(key, default)
|
|
@@ -0,0 +1,175 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "fileutils"
|
|
4
|
+
require_relative "../storage/backup_lock"
|
|
5
|
+
require_relative "../storage/private_file"
|
|
6
|
+
require_relative "../storage/safe_file"
|
|
7
|
+
require_relative "tftp_verification"
|
|
8
|
+
|
|
9
|
+
module Net
|
|
10
|
+
module Connector
|
|
11
|
+
module Netdisco
|
|
12
|
+
# 设备先上传到服务器根目录,核验后按批次归档;固定文件名不会覆盖历史文件。
|
|
13
|
+
class TftpArchive
|
|
14
|
+
BATCH_DIRECTORY_NAME = /\A\d{4}-\d{2}-\d{2}_\d{2}-\d{2}-\d{2}(?:_\d{2})?\z/
|
|
15
|
+
private_constant :BATCH_DIRECTORY_NAME
|
|
16
|
+
|
|
17
|
+
class Unavailable < Net::Connector::Error
|
|
18
|
+
def initialize(host: nil)
|
|
19
|
+
super("a local TFTP root is required to retain a fixed-name upload", code: :tftp_history_unavailable,
|
|
20
|
+
host: host, phase: :tftp_backup)
|
|
21
|
+
end
|
|
22
|
+
end
|
|
23
|
+
|
|
24
|
+
class ArchiveFailed < Net::Connector::Error
|
|
25
|
+
def initialize(host: nil)
|
|
26
|
+
super("the uploaded file could not be retained in the batch", code: :tftp_archive_failed,
|
|
27
|
+
host: host, phase: :tftp_backup)
|
|
28
|
+
end
|
|
29
|
+
end
|
|
30
|
+
|
|
31
|
+
def initialize(directory:, root: nil)
|
|
32
|
+
@report_directory = File.expand_path(directory)
|
|
33
|
+
@server_root = root && File.realpath(root)
|
|
34
|
+
@server_archive_directory = create_server_archive_directory if @server_root
|
|
35
|
+
@remote_suffix = reserve_remote_suffix unless @server_root
|
|
36
|
+
@verifier = TftpVerification.new(root: @server_root)
|
|
37
|
+
end
|
|
38
|
+
|
|
39
|
+
def upload_filename(device)
|
|
40
|
+
name = device.tftp_filename
|
|
41
|
+
return name if @server_root || device.vendor == :palo_alto
|
|
42
|
+
|
|
43
|
+
extension = File.extname(name)
|
|
44
|
+
stem = File.basename(name, extension)
|
|
45
|
+
suffix = "-#{@remote_suffix}#{extension}"
|
|
46
|
+
TftpTarget.validate_path!("#{stem.byteslice(0, TftpTarget::MAX_PATH_BYTES - suffix.bytesize)}#{suffix}")
|
|
47
|
+
end
|
|
48
|
+
|
|
49
|
+
def upload_and_archive(device, started_at:)
|
|
50
|
+
raise Unavailable.new(host: device.host) if device.vendor == :palo_alto && !@server_root
|
|
51
|
+
|
|
52
|
+
remote_filename = upload_filename(device)
|
|
53
|
+
return yield remote_filename unless @server_root
|
|
54
|
+
|
|
55
|
+
# The lock spans the upload and copy, including a fixed-name PAN-OS upload.
|
|
56
|
+
Storage::BackupLock.synchronize(File.join(@server_root, remote_filename), host: device.host, timeout: 240) do
|
|
57
|
+
archive_existing_file(device, remote_filename)
|
|
58
|
+
outcome = yield remote_filename
|
|
59
|
+
receipt = outcome.backup
|
|
60
|
+
return outcome unless receipt.is_a?(TftpReceipt) && receipt.path
|
|
61
|
+
|
|
62
|
+
verified = @verifier.verify_outcome(outcome.with(started_at: started_at))
|
|
63
|
+
return outcome.success? ? archive_failure(outcome) : outcome unless verified.backup.verification == :server_verified
|
|
64
|
+
|
|
65
|
+
archive_uploaded_file(verified)
|
|
66
|
+
end
|
|
67
|
+
end
|
|
68
|
+
|
|
69
|
+
alias path_for upload_filename
|
|
70
|
+
alias capture upload_and_archive
|
|
71
|
+
|
|
72
|
+
private
|
|
73
|
+
|
|
74
|
+
# 首次采用固定远端名时,先保留服务器上原有文件,再允许设备覆盖。
|
|
75
|
+
def archive_existing_file(device, remote_filename)
|
|
76
|
+
existing_path = File.join(@server_root, remote_filename)
|
|
77
|
+
bytes = Storage::SafeFile.open(existing_path, missing: true) { |file, _stat| file.read }
|
|
78
|
+
return unless bytes
|
|
79
|
+
|
|
80
|
+
previous_directory = File.join(@server_archive_directory, "previous")
|
|
81
|
+
FileUtils.mkdir_p(previous_directory, mode: 0o700)
|
|
82
|
+
archive_path = File.join(previous_directory, archive_filename(device, File.extname(remote_filename)))
|
|
83
|
+
Storage::PrivateFile.write(archive_path, bytes)
|
|
84
|
+
rescue StandardError
|
|
85
|
+
raise ArchiveFailed.new(host: device.host), cause: nil
|
|
86
|
+
end
|
|
87
|
+
|
|
88
|
+
def create_server_archive_directory
|
|
89
|
+
parent = File.join(@server_root, "archive")
|
|
90
|
+
raise ArgumentError, "TFTP archive directory must not be a symlink" if File.symlink?(parent)
|
|
91
|
+
|
|
92
|
+
FileUtils.mkdir_p(parent, mode: 0o700)
|
|
93
|
+
raise ArgumentError, "TFTP archive directory escaped its root" unless File.realpath(parent) == parent
|
|
94
|
+
|
|
95
|
+
base = batch_name
|
|
96
|
+
number = 0
|
|
97
|
+
loop do
|
|
98
|
+
candidate = number.zero? ? base : "#{base}_#{format("%02d", number)}"
|
|
99
|
+
path = File.join(parent, candidate)
|
|
100
|
+
begin
|
|
101
|
+
Dir.mkdir(path, 0o700)
|
|
102
|
+
return path
|
|
103
|
+
rescue Errno::EEXIST
|
|
104
|
+
number += 1
|
|
105
|
+
end
|
|
106
|
+
end
|
|
107
|
+
end
|
|
108
|
+
|
|
109
|
+
# 同一目录重跑时用原子预留的序号区分运行,不依赖随机数或进程 ID。
|
|
110
|
+
def reserve_remote_suffix
|
|
111
|
+
base = batch_name
|
|
112
|
+
number = 0
|
|
113
|
+
loop do
|
|
114
|
+
candidate = number.zero? ? base : "#{base}_#{format("%02d", number)}"
|
|
115
|
+
marker = File.join(@report_directory, ".tftp-run-#{candidate}")
|
|
116
|
+
begin
|
|
117
|
+
File.open(marker, File::WRONLY | File::CREAT | File::EXCL, 0o600) { |file| file.write("\n") }
|
|
118
|
+
return candidate
|
|
119
|
+
rescue Errno::EEXIST
|
|
120
|
+
number += 1
|
|
121
|
+
end
|
|
122
|
+
end
|
|
123
|
+
end
|
|
124
|
+
|
|
125
|
+
def batch_name
|
|
126
|
+
name = File.basename(@report_directory)
|
|
127
|
+
name.match?(BATCH_DIRECTORY_NAME) ? name : Time.now.getlocal("+08:00").strftime("%Y-%m-%d_%H-%M-%S")
|
|
128
|
+
end
|
|
129
|
+
|
|
130
|
+
def archive_uploaded_file(outcome)
|
|
131
|
+
receipt = outcome.backup
|
|
132
|
+
saved_batch_path = nil
|
|
133
|
+
batch_directory = File.join(@report_directory, "tftp")
|
|
134
|
+
FileUtils.mkdir_p(batch_directory, mode: 0o700)
|
|
135
|
+
extension = File.extname(receipt.path)
|
|
136
|
+
filename = archive_filename(outcome.device, extension)
|
|
137
|
+
batch_archive_path = File.join(batch_directory, filename)
|
|
138
|
+
server_archive_path = File.join(@server_archive_directory, filename)
|
|
139
|
+
uploaded_path = receipt.local_path
|
|
140
|
+
bytes = Storage::SafeFile.open(uploaded_path) do |file, stat|
|
|
141
|
+
raise ArchiveFailed.new(host: outcome.device.host) unless stat.size == receipt.server_bytes
|
|
142
|
+
|
|
143
|
+
file.read
|
|
144
|
+
end
|
|
145
|
+
raise ArchiveFailed.new(host: outcome.device.host) unless Digest::SHA256.hexdigest(bytes) == receipt.server_sha256
|
|
146
|
+
|
|
147
|
+
Storage::PrivateFile.write(batch_archive_path, bytes)
|
|
148
|
+
saved_batch_path = batch_archive_path
|
|
149
|
+
Storage::PrivateFile.write(server_archive_path, bytes)
|
|
150
|
+
if outcome.success?
|
|
151
|
+
uploaded_fingerprint = Storage::SafeFile.fingerprint(uploaded_path)
|
|
152
|
+
raise ArchiveFailed.new(host: outcome.device.host) unless uploaded_fingerprint.sha256 == receipt.server_sha256
|
|
153
|
+
raise ArchiveFailed.new(host: outcome.device.host) unless Storage::SafeFile.same_entry?(uploaded_path, uploaded_fingerprint)
|
|
154
|
+
|
|
155
|
+
File.unlink(uploaded_path)
|
|
156
|
+
end
|
|
157
|
+
outcome.with(backup: receipt.with(local_path: batch_archive_path, archive_path: server_archive_path))
|
|
158
|
+
rescue StandardError
|
|
159
|
+
retained = outcome.with(backup: receipt.with(local_path: saved_batch_path))
|
|
160
|
+
outcome.success? ? archive_failure(retained) : retained
|
|
161
|
+
end
|
|
162
|
+
|
|
163
|
+
def archive_failure(outcome)
|
|
164
|
+
error = ArchiveFailed.new(host: outcome.device.host)
|
|
165
|
+
outcome.with(status: :reported_with_error, error_code: error.code, error_type: error.class.name,
|
|
166
|
+
diagnostic: Diagnostic.from(error, backup: outcome.backup))
|
|
167
|
+
end
|
|
168
|
+
|
|
169
|
+
def archive_filename(device, extension)
|
|
170
|
+
"#{device.backup_filename(style: :hostname_ip).delete_suffix(".txt")}#{extension}"
|
|
171
|
+
end
|
|
172
|
+
end
|
|
173
|
+
end
|
|
174
|
+
end
|
|
175
|
+
end
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require_relative "tftp_archive"
|
|
4
|
+
|
|
5
|
+
module Net
|
|
6
|
+
module Connector
|
|
7
|
+
module Netdisco
|
|
8
|
+
# Compatibility with the first batch archive entry point.
|
|
9
|
+
class TftpHistory < TftpArchive
|
|
10
|
+
def initialize(server:, **options) # rubocop:disable Lint/UnusedMethodArgument -- Accept the former constructor keyword.
|
|
11
|
+
super(**options)
|
|
12
|
+
end
|
|
13
|
+
end
|
|
14
|
+
end
|
|
15
|
+
end
|
|
16
|
+
end
|
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require_relative "../storage/safe_file"
|
|
4
|
+
|
|
5
|
+
module Net
|
|
6
|
+
module Connector
|
|
7
|
+
module Netdisco
|
|
8
|
+
# 只核验本次设备回执指向的本地服务器文件;远端可读不等于本次上传证据。
|
|
9
|
+
class TftpVerification
|
|
10
|
+
def initialize(root: nil)
|
|
11
|
+
@root = root && File.realpath(root)
|
|
12
|
+
end
|
|
13
|
+
|
|
14
|
+
def call(report)
|
|
15
|
+
return report unless @root
|
|
16
|
+
|
|
17
|
+
outcomes = report.outcomes.map do |outcome|
|
|
18
|
+
receipt = outcome.backup
|
|
19
|
+
next outcome unless receipt.is_a?(TftpReceipt) && receipt.path && outcome.started_at
|
|
20
|
+
|
|
21
|
+
verify(outcome, receipt)
|
|
22
|
+
end
|
|
23
|
+
report.with(outcomes: outcomes.freeze)
|
|
24
|
+
end
|
|
25
|
+
|
|
26
|
+
def verify_outcome(outcome)
|
|
27
|
+
return outcome unless @root && outcome.started_at && outcome.backup.is_a?(TftpReceipt) && outcome.backup.path
|
|
28
|
+
|
|
29
|
+
verify(outcome, outcome.backup)
|
|
30
|
+
end
|
|
31
|
+
|
|
32
|
+
private
|
|
33
|
+
|
|
34
|
+
def verify(outcome, receipt)
|
|
35
|
+
path = File.join(@root, receipt.path)
|
|
36
|
+
parent = File.realpath(File.dirname(path))
|
|
37
|
+
return outcome unless parent == @root || parent.start_with?(@root + File::SEPARATOR)
|
|
38
|
+
|
|
39
|
+
size = nil
|
|
40
|
+
fingerprint = Storage::SafeFile.open(path, missing: true) do |file, stat|
|
|
41
|
+
next unless stat.size.positive? && stat.mtime >= outcome.started_at
|
|
42
|
+
|
|
43
|
+
size = stat.size
|
|
44
|
+
value = Storage::SafeFile.fingerprint_io(path, file, stat)
|
|
45
|
+
after = file.stat
|
|
46
|
+
value if [stat.size, stat.mtime, stat.ctime] == [after.size, after.mtime, after.ctime]
|
|
47
|
+
end
|
|
48
|
+
return outcome unless fingerprint && Storage::SafeFile.same_entry?(path, fingerprint)
|
|
49
|
+
|
|
50
|
+
outcome.with(backup: receipt.with(verification: :server_verified, server_sha256: fingerprint.sha256, server_bytes: size, local_path: path))
|
|
51
|
+
rescue IOError, SystemCallError, ArgumentError
|
|
52
|
+
# 核验失败不抹去设备上传回执;要求服务器证据的策略仍判为未完成。
|
|
53
|
+
outcome
|
|
54
|
+
end
|
|
55
|
+
end
|
|
56
|
+
end
|
|
57
|
+
end
|
|
58
|
+
end
|
|
@@ -5,13 +5,18 @@ module Net
|
|
|
5
5
|
module Netdisco
|
|
6
6
|
# 以固定并发上限执行相互隔离的设备任务。
|
|
7
7
|
class Worker
|
|
8
|
-
#
|
|
9
|
-
def
|
|
8
|
+
# 内部预检与实际执行共用规则,不为校验创建工作线程池对象。
|
|
9
|
+
def self.validate_concurrency!(concurrency)
|
|
10
10
|
unless concurrency.is_a?(Integer) && (1..Settings::MAX_CONCURRENCY).cover?(concurrency)
|
|
11
11
|
raise ArgumentError, "concurrency must be an Integer in 1..#{Settings::MAX_CONCURRENCY}"
|
|
12
12
|
end
|
|
13
13
|
|
|
14
|
-
|
|
14
|
+
concurrency
|
|
15
|
+
end
|
|
16
|
+
|
|
17
|
+
# 校验并保存最大设备并发数。
|
|
18
|
+
def initialize(concurrency:)
|
|
19
|
+
@concurrency = Worker.validate_concurrency!(concurrency)
|
|
15
20
|
end
|
|
16
21
|
|
|
17
22
|
# 按清单顺序收集结果,并隔离设备及回调异常。
|
|
@@ -12,4 +12,14 @@ require_relative "netdisco/result_store"
|
|
|
12
12
|
require_relative "netdisco/batch"
|
|
13
13
|
require_relative "netdisco/planner"
|
|
14
14
|
require_relative "netdisco/fleet"
|
|
15
|
+
require_relative "netdisco/progress"
|
|
15
16
|
require_relative "netdisco/cli"
|
|
17
|
+
require_relative "storage/batch_directory"
|
|
18
|
+
require_relative "netdisco/report/text"
|
|
19
|
+
require_relative "netdisco/connection"
|
|
20
|
+
require_relative "netdisco/cli/options"
|
|
21
|
+
require_relative "netdisco/report/files"
|
|
22
|
+
require_relative "netdisco/tftp_verification"
|
|
23
|
+
require_relative "netdisco/tftp_archive"
|
|
24
|
+
require_relative "netdisco/tftp_history"
|
|
25
|
+
require_relative "netdisco/backup_run"
|
|
@@ -25,7 +25,7 @@ module Net
|
|
|
25
25
|
File.join(directory, ".net-connector-#{Digest::SHA256.hexdigest(filename)}.lock")
|
|
26
26
|
end
|
|
27
27
|
|
|
28
|
-
def self.synchronize(path, **options, &
|
|
28
|
+
def self.synchronize(path, **options, &) = new(path, **options).synchronize(&)
|
|
29
29
|
|
|
30
30
|
def initialize(path, timeout: 0, host: nil)
|
|
31
31
|
raise ArgumentError, "path must be a nonempty String" unless path.is_a?(String) && !path.empty?
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "fileutils"
|
|
4
|
+
|
|
5
|
+
module Net
|
|
6
|
+
module Connector
|
|
7
|
+
module Storage
|
|
8
|
+
# 使用本地业务时区命名,原子创建互不覆盖的批次目录。
|
|
9
|
+
class BatchDirectory
|
|
10
|
+
# 固定 UTC+8,不依赖运行主机时区;mkdir 原子防止并发任务覆盖。
|
|
11
|
+
def self.create(root, time: Time.now)
|
|
12
|
+
FileUtils.mkdir_p(root, mode: 0o700)
|
|
13
|
+
stamp = time.getlocal("+08:00").strftime("%Y-%m-%d_%H-%M-%S")
|
|
14
|
+
suffix = 0
|
|
15
|
+
loop do
|
|
16
|
+
name = suffix.zero? ? stamp : "#{stamp}_#{format("%02d", suffix)}"
|
|
17
|
+
path = File.join(root, name)
|
|
18
|
+
begin
|
|
19
|
+
Dir.mkdir(path, 0o700)
|
|
20
|
+
return path
|
|
21
|
+
rescue Errno::EEXIST
|
|
22
|
+
suffix += 1
|
|
23
|
+
end
|
|
24
|
+
end
|
|
25
|
+
end
|
|
26
|
+
end
|
|
27
|
+
end
|
|
28
|
+
end
|
|
29
|
+
end
|
|
@@ -4,6 +4,7 @@ module Net
|
|
|
4
4
|
module Connector
|
|
5
5
|
# 文件存储不依赖设备连接;备份、离线导出和批量报告共享这些原语。
|
|
6
6
|
module Storage
|
|
7
|
+
autoload :BatchDirectory, File.expand_path("storage/batch_directory", __dir__)
|
|
7
8
|
autoload :BackupLock, File.expand_path("storage/backup_lock", __dir__)
|
|
8
9
|
autoload :PrivateFile, File.expand_path("storage/private_file", __dir__)
|
|
9
10
|
autoload :SafeFile, File.expand_path("storage/safe_file", __dir__)
|