net-connector 0.6.0 → 0.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +52 -0
- data/CONTRIBUTING.md +20 -0
- data/README.md +112 -50
- data/docs/VERIFICATION.md +42 -1
- data/docs/architecture.md +15 -2
- data/examples/backup.yml +40 -0
- data/lib/net/connector/device/base.rb +30 -38
- data/lib/net/connector/device/profile/builder.rb +8 -8
- data/lib/net/connector/device/profile.rb +2 -2
- data/lib/net/connector/device/running_config.rb +2 -2
- data/lib/net/connector/device/tftp/receipt.rb +12 -3
- data/lib/net/connector/device/tftp.rb +7 -7
- data/lib/net/connector/device/topology.rb +4 -4
- data/lib/net/connector/engine/command.rb +2 -2
- data/lib/net/connector/engine/configuration.rb +5 -2
- data/lib/net/connector/engine/execution.rb +25 -14
- data/lib/net/connector/engine/known_hosts.rb +112 -0
- data/lib/net/connector/engine/log.rb +10 -7
- data/lib/net/connector/engine/session.rb +117 -128
- data/lib/net/connector/engine/transport.rb +32 -7
- data/lib/net/connector/netdisco/backup_run.rb +132 -0
- data/lib/net/connector/netdisco/batch.rb +1 -1
- data/lib/net/connector/netdisco/cli/options.rb +139 -0
- data/lib/net/connector/netdisco/cli.rb +13 -26
- data/lib/net/connector/netdisco/client.rb +6 -6
- data/lib/net/connector/netdisco/config_file.rb +20 -19
- data/lib/net/connector/netdisco/connection.rb +38 -0
- data/lib/net/connector/netdisco/database_client.rb +2 -2
- data/lib/net/connector/netdisco/device.rb +11 -1
- data/lib/net/connector/netdisco/fleet.rb +68 -47
- data/lib/net/connector/netdisco/inventory_budget.rb +4 -4
- data/lib/net/connector/netdisco/plan.rb +14 -9
- data/lib/net/connector/netdisco/planner.rb +18 -14
- data/lib/net/connector/netdisco/progress.rb +253 -0
- data/lib/net/connector/netdisco/report/files.rb +41 -0
- data/lib/net/connector/netdisco/report/text.rb +45 -0
- data/lib/net/connector/netdisco/report.rb +53 -31
- data/lib/net/connector/netdisco/result_store.rb +3 -1
- data/lib/net/connector/netdisco/rules.rb +10 -2
- data/lib/net/connector/netdisco/settings.rb +118 -68
- data/lib/net/connector/netdisco/tftp_archive.rb +175 -0
- data/lib/net/connector/netdisco/tftp_history.rb +16 -0
- data/lib/net/connector/netdisco/tftp_verification.rb +58 -0
- data/lib/net/connector/netdisco/worker.rb +8 -3
- data/lib/net/connector/netdisco.rb +10 -0
- data/lib/net/connector/storage/backup_lock.rb +1 -1
- data/lib/net/connector/storage/batch_directory.rb +29 -0
- data/lib/net/connector/storage.rb +1 -0
- data/lib/net/connector/vendor/h3c.rb +2 -1
- data/lib/net/connector/vendor/hillstone/tftp_backup.rb +3 -3
- data/lib/net/connector/vendor/palo_alto/tftp_backup.rb +3 -3
- data/lib/net/connector/vendor/radware/running_config.rb +19 -0
- data/lib/net/connector/vendor/radware.rb +4 -3
- data/lib/net/connector/version.rb +1 -1
- data/lib/net/connector.rb +2 -2
- metadata +42 -111
- /data/examples/{netdisco_database.yml → inventory_sql.yml} +0 -0
|
@@ -54,7 +54,10 @@ module Net
|
|
|
54
54
|
message = Storage::PrivateFile.receipt_error?(exception) ? exception.message : exception.class.name
|
|
55
55
|
@error.puts "net-connector-backup: #{message}"
|
|
56
56
|
2
|
|
57
|
-
rescue OptionParser::ParseError
|
|
57
|
+
rescue OptionParser::ParseError
|
|
58
|
+
@error.puts "net-connector-backup: 参数无效;使用 --help 查看参数"
|
|
59
|
+
2
|
|
60
|
+
rescue ArgumentError, KeyError, Errno::ENOENT,
|
|
58
61
|
Psych::Exception, JSON::ParserError, Client::Error => exception
|
|
59
62
|
@error.puts "net-connector-backup: #{exception.message}"
|
|
60
63
|
2
|
|
@@ -67,18 +70,7 @@ module Net
|
|
|
67
70
|
|
|
68
71
|
# 文件、环境变量、命令行按优先级覆盖,凭据仍只来自环境变量。
|
|
69
72
|
def settings_for(options)
|
|
70
|
-
|
|
71
|
-
values = {}
|
|
72
|
-
values["NETDISCO_SOURCE"] = options[:source] if options[:source]
|
|
73
|
-
values["NETDISCO_QUERY"] = options[:query] if options[:query]
|
|
74
|
-
values["NETDISCO_QUERY_PARAMS"] = options[:query_params] if options[:query_params]
|
|
75
|
-
values["NET_CONNECTOR_BACKUP_DIRECTORY"] = options[:directory] if options[:directory]
|
|
76
|
-
values["NET_CONNECTOR_CONCURRENCY"] = options[:concurrency].to_s if options[:concurrency]
|
|
77
|
-
values["NET_CONNECTOR_MAX_SCRIPT_OUTPUT_BYTES"] = options[:max_script_output_bytes].to_s if options[:max_script_output_bytes]
|
|
78
|
-
values["NET_CONNECTOR_INCLUDE_HOSTS"] = options[:host] if options[:host]
|
|
79
|
-
values["NET_CONNECTOR_SAMPLE_PER_VENDOR"] = options[:limit].to_s if options[:limit]
|
|
80
|
-
values["NET_CONNECTOR_SAMPLE_PER_VENDOR"] = nil if options[:all]
|
|
81
|
-
Settings.new(env: @env, defaults: defaults, overrides: values)
|
|
73
|
+
Options.settings(options, env: @env)
|
|
82
74
|
end
|
|
83
75
|
|
|
84
76
|
# 导出已有配置不创建清单或设备连接。
|
|
@@ -110,7 +102,7 @@ module Net
|
|
|
110
102
|
fleet.tftp_backup_all(plan: plan, server: settings.tftp_server,
|
|
111
103
|
concurrency: settings.concurrency,
|
|
112
104
|
vrfs: settings.tftp_vrfs,
|
|
113
|
-
source_files: settings.tftp_source_files, **reporting)
|
|
105
|
+
source_files: settings.tftp_source_files, verification_root: options[:tftp_root] || @env["TFTP_ROOT"], **reporting)
|
|
114
106
|
else
|
|
115
107
|
fleet.backup_all(plan: plan, directory: settings.backup_directory,
|
|
116
108
|
concurrency: settings.concurrency, **reporting)
|
|
@@ -119,10 +111,11 @@ module Net
|
|
|
119
111
|
|
|
120
112
|
# 解析命令行参数并处理帮助及版本信息。
|
|
121
113
|
def parse_options
|
|
122
|
-
options = {
|
|
114
|
+
options = { environment: {} }
|
|
123
115
|
parser = OptionParser.new do |args|
|
|
124
116
|
args.banner = "用法:net-connector-backup [--config FILE] [--plan | --show-config | --export IP] [--tftp]"
|
|
125
|
-
|
|
117
|
+
Options.add_settings(args, options)
|
|
118
|
+
Options.add_execution(args, options)
|
|
126
119
|
args.on("--source SOURCE", %w[http postgres], "清单来源:http(默认)或 postgres") { |value| options[:source] = value }
|
|
127
120
|
args.on("--query SQL", "PostgreSQL 清单查询,使用 $1、$2 绑定参数") { |value| options[:query] = value }
|
|
128
121
|
args.on("--query-params JSON", "查询参数的 JSON 数组") { |value| options[:query_params] = value }
|
|
@@ -131,15 +124,8 @@ module Net
|
|
|
131
124
|
args.on("--tftp", "由设备发起 TFTP 备份") { options[:tftp] = true }
|
|
132
125
|
args.on("--export IP", "导出已保存的本地配置") { |value| options[:export] = value }
|
|
133
126
|
args.on("--output FILE", "将导出配置写入私有文件") { |value| options[:output] = value }
|
|
134
|
-
args.on("--directory PATH", "备份和报告目录") { |value| options[:directory] = value }
|
|
135
127
|
args.on("--host IP", "按管理地址选择一台设备") { |value| options[:host] = value }
|
|
136
|
-
args.on("--concurrency N", Integer, "最大并发设备数") { |value| options[:concurrency] = value }
|
|
137
128
|
args.on("--max-script-output-bytes N", Integer, "每个脚本的累计响应字节上限") { |value| options[:max_script_output_bytes] = value }
|
|
138
|
-
args.on("--limit-per-vendor N", Integer, "每厂商选择 N 台设备(1 至 5)") do |value|
|
|
139
|
-
options[:limit] = value
|
|
140
|
-
end
|
|
141
|
-
args.on("--all", "选择全部就绪设备") { options[:all] = true }
|
|
142
|
-
args.on("--success-policy POLICY", %w[strict selected], "成功判定:strict(默认)或 selected") { |value| options[:success_policy] = value.to_sym }
|
|
143
129
|
args.on("-v", "--version", "显示版本") do
|
|
144
130
|
@output.puts Net::Connector::VERSION
|
|
145
131
|
options[:done] = true
|
|
@@ -157,9 +143,10 @@ module Net
|
|
|
157
143
|
|
|
158
144
|
# 拒绝互斥或缺少依赖的命令行选项。
|
|
159
145
|
def validate_options!(options)
|
|
146
|
+
raise ArgumentError, "verified policy requires --tftp" if options[:success_policy] == :verified && !options[:tftp]
|
|
160
147
|
Report.validate_policy!(options.fetch(:success_policy, :strict))
|
|
161
148
|
Planner.validate_limit!(options[:limit])
|
|
162
|
-
|
|
149
|
+
raise ArgumentError, "--tftp-root requires --tftp" if options[:tftp_root] && !options[:tftp]
|
|
163
150
|
modes = [:show_config, :plan, :export].count { |key| options[key] }
|
|
164
151
|
raise ArgumentError, "--show-config、--plan 和 --export 只能选择一项" if modes > 1
|
|
165
152
|
raise ArgumentError, "--output 需要同时指定 --export" if options[:output] && !options[:export]
|
|
@@ -167,8 +154,8 @@ module Net
|
|
|
167
154
|
if options[:export] && (options[:source] || options[:query] || options[:query_params])
|
|
168
155
|
raise ArgumentError, "--export 不能与清单查询选项同时使用"
|
|
169
156
|
end
|
|
170
|
-
if options[:export] && (options[:tftp] || options[:all] || options[:limit] || options[:
|
|
171
|
-
options[:success_policy] || options[:max_script_output_bytes])
|
|
157
|
+
if options[:export] && (options[:tftp] || options[:all] || options[:limit] || options[:environment].key?("NC_CONCURRENCY") || options[:host] ||
|
|
158
|
+
options[:success_policy] || options[:tftp_root] || options[:max_script_output_bytes])
|
|
172
159
|
raise ArgumentError, "--export 不能与备份选项同时使用"
|
|
173
160
|
end
|
|
174
161
|
end
|
|
@@ -258,9 +258,9 @@ module Net
|
|
|
258
258
|
row.is_a?(Hash) && present?(row["ip"]) &&
|
|
259
259
|
FIELDS.drop(1).all? { |field| row[field].nil? || row[field].is_a?(String) }
|
|
260
260
|
end
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
261
|
+
return if valid
|
|
262
|
+
|
|
263
|
+
raise Error.new("Netdisco returned an invalid device inventory", code: :invalid_inventory), cause: nil
|
|
264
264
|
end
|
|
265
265
|
|
|
266
266
|
def validate_credentials!(username, password, api_key)
|
|
@@ -268,9 +268,9 @@ module Net
|
|
|
268
268
|
(present?(api_key) && username.nil? && password.nil?)
|
|
269
269
|
raise ArgumentError, "provide either username and password or api_key"
|
|
270
270
|
end
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
271
|
+
return unless [username, password, api_key].compact.any? { |value| value.match?(/[\r\n\x00]/) }
|
|
272
|
+
|
|
273
|
+
raise ArgumentError, "credentials must be single-line strings"
|
|
274
274
|
end
|
|
275
275
|
|
|
276
276
|
def monotonic = Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
|
@@ -20,26 +20,27 @@ module Net
|
|
|
20
20
|
"max_devices" => ["NETDISCO_MAX_DEVICES", :integer],
|
|
21
21
|
"inventory_timeout" => ["NETDISCO_INVENTORY_TIMEOUT", :number],
|
|
22
22
|
"allow_insecure_http" => ["NETDISCO_ALLOW_INSECURE_HTTP", :boolean] },
|
|
23
|
-
"backup" => { "directory" => ["
|
|
24
|
-
"concurrency" => ["
|
|
25
|
-
"limit_per_vendor" => ["
|
|
26
|
-
"inventory" => { "include_hosts" => ["
|
|
27
|
-
"exclude_hosts" => ["
|
|
28
|
-
"include_vendors" => ["
|
|
29
|
-
"vendor_overrides" => ["
|
|
30
|
-
"host_overrides" => ["
|
|
31
|
-
"device_rules" => ["
|
|
32
|
-
"ssh" => { "protocol" => ["
|
|
33
|
-
"
|
|
34
|
-
"
|
|
35
|
-
"
|
|
36
|
-
"
|
|
37
|
-
"
|
|
23
|
+
"backup" => { "directory" => ["NC_BACKUP_DIRECTORY", :string],
|
|
24
|
+
"concurrency" => ["NC_CONCURRENCY", :integer],
|
|
25
|
+
"limit_per_vendor" => ["NC_SAMPLE_PER_VENDOR", :integer] },
|
|
26
|
+
"inventory" => { "include_hosts" => ["NC_INCLUDE_HOSTS", :list],
|
|
27
|
+
"exclude_hosts" => ["NC_EXCLUDE_HOSTS", :list],
|
|
28
|
+
"include_vendors" => ["NC_INCLUDE_VENDORS", :list],
|
|
29
|
+
"vendor_overrides" => ["NC_VENDOR_OVERRIDES", :json],
|
|
30
|
+
"host_overrides" => ["NC_HOST_OVERRIDES", :json],
|
|
31
|
+
"device_rules" => ["NC_DEVICE_RULES", :json] },
|
|
32
|
+
"ssh" => { "protocol" => ["NC_PROTOCOL", :string],
|
|
33
|
+
"vendor_protocols" => ["NC_VENDOR_PROTOCOLS", :json],
|
|
34
|
+
"max_script_output_bytes" => ["NC_MAX_SCRIPT_OUTPUT_BYTES", :optional_integer],
|
|
35
|
+
"known_hosts" => ["NC_KNOWN_HOSTS", :string],
|
|
36
|
+
"host_key_policy" => ["NC_HOST_KEY_POLICY", :string],
|
|
37
|
+
"log_directory" => ["NC_LOG_DIRECTORY", :string],
|
|
38
|
+
"log_level" => ["NC_LOG_LEVEL", :string] },
|
|
38
39
|
"tftp" => { "server" => ["TFTP_HOST", :string],
|
|
39
|
-
"vrfs" => ["
|
|
40
|
-
"h3c_source_file" => ["
|
|
41
|
-
"h3c_wireless_source_file" => ["
|
|
42
|
-
"huawei_source_file" => ["
|
|
40
|
+
"vrfs" => ["NC_TFTP_VRFS", :json],
|
|
41
|
+
"h3c_source_file" => ["NC_H3C_TFTP_SOURCE_FILE", :string],
|
|
42
|
+
"h3c_wireless_source_file" => ["NC_H3C_WIRELESS_TFTP_SOURCE_FILE", :string],
|
|
43
|
+
"huawei_source_file" => ["NC_HUAWEI_TFTP_SOURCE_FILE", :string] }
|
|
43
44
|
}.freeze
|
|
44
45
|
|
|
45
46
|
# 安全读取 YAML 配置并映射到环境变量格式。
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "json"
|
|
4
|
+
require_relative "client"
|
|
5
|
+
|
|
6
|
+
# 清单连接与设备凭据来源;业务入口负责规划和执行备份。
|
|
7
|
+
module Net
|
|
8
|
+
module Connector
|
|
9
|
+
module Netdisco
|
|
10
|
+
# 将调用方提供的输入解析为清单客户端和设备凭据 resolver。
|
|
11
|
+
class Connection
|
|
12
|
+
def self.build(settings, stdin_credentials: false, input: $stdin)
|
|
13
|
+
unless stdin_credentials
|
|
14
|
+
return [settings.client, settings.method(:device_credentials_for)]
|
|
15
|
+
end
|
|
16
|
+
|
|
17
|
+
begin
|
|
18
|
+
values = JSON.parse(input.gets || "")
|
|
19
|
+
rescue JSON::ParserError
|
|
20
|
+
raise ArgumentError, "凭据 JSON 无效", cause: nil
|
|
21
|
+
end
|
|
22
|
+
keys = %w[netdisco_username netdisco_password device_username device_password]
|
|
23
|
+
raise ArgumentError, "需要四个非空凭据字符串" unless values.is_a?(Hash) && keys.all? { |key| values[key].is_a?(String) && !values[key].strip.empty? }
|
|
24
|
+
|
|
25
|
+
client = settings.client(credentials: {
|
|
26
|
+
username: values.fetch("netdisco_username"), password: values.fetch("netdisco_password")
|
|
27
|
+
})
|
|
28
|
+
credentials = lambda do |device|
|
|
29
|
+
inherited = device ? settings.device_credentials_for(device) : nil
|
|
30
|
+
(inherited || {}).merge(username: values.fetch("device_username"),
|
|
31
|
+
password: values.fetch("device_password"))
|
|
32
|
+
end
|
|
33
|
+
[client, credentials]
|
|
34
|
+
end
|
|
35
|
+
end
|
|
36
|
+
end
|
|
37
|
+
end
|
|
38
|
+
end
|
|
@@ -37,7 +37,7 @@ module Net
|
|
|
37
37
|
|
|
38
38
|
def initialize(connection_options:, **options)
|
|
39
39
|
@options = self.class.options(**options)
|
|
40
|
-
@connection_options = validate_connection_options(connection_options)
|
|
40
|
+
@connection_options = validate_connection_options!(connection_options)
|
|
41
41
|
end
|
|
42
42
|
|
|
43
43
|
# 每次查询独占连接和游标;只有全部结果验证成功后才把清单交给 Fleet。
|
|
@@ -103,7 +103,7 @@ module Net
|
|
|
103
103
|
end
|
|
104
104
|
end
|
|
105
105
|
|
|
106
|
-
def validate_connection_options(options)
|
|
106
|
+
def validate_connection_options!(options)
|
|
107
107
|
unless options.is_a?(Hash) && (options.keys - CONNECTION_ENV.keys).empty?
|
|
108
108
|
raise ArgumentError, "connection_options must contain supported PostgreSQL connection keys"
|
|
109
109
|
end
|
|
@@ -44,7 +44,17 @@ module Net
|
|
|
44
44
|
def ready? = issue.nil?
|
|
45
45
|
|
|
46
46
|
# 生成不随名称或厂商信息改变的本地文本文件名。
|
|
47
|
-
def backup_filename
|
|
47
|
+
def backup_filename(style: :ip)
|
|
48
|
+
address = Storage::SavedConfig.filename(host)
|
|
49
|
+
return address if style == :ip
|
|
50
|
+
raise ArgumentError, "filename style must be ip or hostname_ip" unless style == :hostname_ip
|
|
51
|
+
|
|
52
|
+
label = name.to_s.encode("UTF-8", invalid: :replace, undef: :replace, replace: "-")
|
|
53
|
+
.gsub(/[^\p{L}\p{N}._-]+/, "-").byteslice(0, 160).scrub("")
|
|
54
|
+
.gsub(/\A[._-]+|[._-]+\z/, "")
|
|
55
|
+
label = "unnamed" if label.empty?
|
|
56
|
+
"#{label}-#{address}"
|
|
57
|
+
end
|
|
48
58
|
|
|
49
59
|
# 生成设备 CLI 可用的 ASCII 远端文件名。
|
|
50
60
|
def tftp_filename
|
|
@@ -13,7 +13,7 @@ module Net
|
|
|
13
13
|
|
|
14
14
|
# 组装清单客户端、规则、凭据与结果存储。
|
|
15
15
|
def initialize(client: nil, settings: Settings.new, rules: nil, credentials: nil, connector_factory: nil,
|
|
16
|
-
result_store: ResultStore::
|
|
16
|
+
result_store: ResultStore::Json.new)
|
|
17
17
|
@settings = settings
|
|
18
18
|
@client = client
|
|
19
19
|
@rules = rules
|
|
@@ -35,46 +35,53 @@ module Net
|
|
|
35
35
|
end
|
|
36
36
|
|
|
37
37
|
# 规划 TFTP 配置备份的设备集合。
|
|
38
|
-
def plan_tftp_backup(limit_per_vendor: 5)
|
|
39
|
-
build_plan(:tftp, limit_per_vendor, @settings.snapshot(mode: :tftp)
|
|
38
|
+
def plan_tftp_backup(limit_per_vendor: 5, allow_fixed_name_reuse: false)
|
|
39
|
+
build_plan(:tftp, limit_per_vendor, @settings.snapshot(mode: :tftp),
|
|
40
|
+
allow_fixed_name_reuse: allow_fixed_name_reuse)
|
|
40
41
|
end
|
|
41
42
|
|
|
42
43
|
# 先拉取完整清单,再并发采集并保存设备配置。
|
|
43
44
|
def backup_all(directory: DEFAULT_SETTING, concurrency: DEFAULT_SETTING,
|
|
44
45
|
limit_per_vendor: nil, plan: nil, on_start: nil, on_result: nil, on_change: nil,
|
|
45
|
-
success_policy: :strict)
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
46
|
+
success_policy: :strict, filename_style: :ip)
|
|
47
|
+
raise ArgumentError, "filename style must be ip or hostname_ip" unless %i[ip hostname_ip].include?(filename_style)
|
|
48
|
+
|
|
49
|
+
Report.validate_policy!(success_policy)
|
|
50
|
+
raise ArgumentError, "verified policy requires TFTP mode" if success_policy == :verified
|
|
51
|
+
settings_snapshot = @settings.snapshot(mode: :backup)
|
|
52
|
+
directory = settings_snapshot.backup_directory if directory.equal?(DEFAULT_SETTING)
|
|
53
|
+
concurrency = settings_snapshot.concurrency if concurrency.equal?(DEFAULT_SETTING)
|
|
50
54
|
raise ArgumentError, "directory must be a nonempty String" unless directory.is_a?(String) && !directory.empty?
|
|
51
|
-
Worker.
|
|
55
|
+
Worker.validate_concurrency!(concurrency)
|
|
52
56
|
Planner.validate_limit!(limit_per_vendor)
|
|
53
57
|
validate_callback!(on_result, :on_result)
|
|
54
58
|
validate_callback!(on_start, :on_start)
|
|
55
59
|
validate_callback!(on_change, :on_change)
|
|
56
60
|
|
|
57
|
-
plan ||= build_plan(:backup, limit_per_vendor,
|
|
61
|
+
plan ||= build_plan(:backup, limit_per_vendor, settings_snapshot)
|
|
58
62
|
validate_plan!(plan, :backup)
|
|
59
63
|
callbacks = [on_result]
|
|
60
|
-
callbacks << ->(
|
|
64
|
+
callbacks << ->(outcome) { on_change.call(outcome) if outcome.backup.is_a?(Backup) && outcome.backup.changed? } if on_change
|
|
61
65
|
target_directory = File.expand_path(directory)
|
|
62
66
|
run_batch(:backup, plan, concurrency: concurrency, report_directory: directory,
|
|
63
|
-
output_directory: target_directory,
|
|
67
|
+
output_directory: target_directory, settings_snapshot: settings_snapshot, success_policy: success_policy,
|
|
64
68
|
on_start: on_start, on_result: callbacks.compact) do |device, log_directory|
|
|
65
|
-
|
|
69
|
+
backup_device(device, target_directory, log_directory, settings_snapshot, filename_style: filename_style)
|
|
66
70
|
end
|
|
67
71
|
end
|
|
68
72
|
|
|
69
73
|
# 让设备主动导出配置,在执行前检查同批目标文件名冲突。
|
|
70
74
|
def tftp_backup_all(server:, source_files: {}, concurrency: DEFAULT_SETTING,
|
|
71
75
|
limit_per_vendor: 5, vrfs: {}, on_start: nil, on_result: nil, plan: nil,
|
|
72
|
-
report_directory: DEFAULT_SETTING, success_policy: :strict
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
76
|
+
report_directory: DEFAULT_SETTING, success_policy: :strict, verification_root: nil,
|
|
77
|
+
preserve_history: false)
|
|
78
|
+
Report.validate_policy!(success_policy)
|
|
79
|
+
raise ArgumentError, "verified policy requires a local TFTP directory" if success_policy == :verified && !verification_root
|
|
80
|
+
verification = TftpVerification.new(root: verification_root)
|
|
81
|
+
settings_snapshot = @settings.snapshot(mode: :tftp)
|
|
82
|
+
concurrency = settings_snapshot.concurrency if concurrency.equal?(DEFAULT_SETTING)
|
|
83
|
+
report_directory = settings_snapshot.backup_directory if report_directory.equal?(DEFAULT_SETTING)
|
|
84
|
+
Worker.validate_concurrency!(concurrency)
|
|
78
85
|
server = TftpTarget.new(host: server, path: "preflight.cfg").host
|
|
79
86
|
raise ArgumentError, "source_files must be a Hash" unless source_files.is_a?(Hash)
|
|
80
87
|
source_files.each_value { |source_file| TftpTarget.validate_source_file!(source_file) }
|
|
@@ -84,40 +91,53 @@ module Net
|
|
|
84
91
|
validate_callback!(on_result, :on_result)
|
|
85
92
|
validate_callback!(on_start, :on_start)
|
|
86
93
|
|
|
87
|
-
plan ||= build_plan(:tftp, limit_per_vendor,
|
|
94
|
+
plan ||= build_plan(:tftp, limit_per_vendor, settings_snapshot,
|
|
95
|
+
allow_fixed_name_reuse: preserve_history && !verification_root.nil?)
|
|
88
96
|
validate_plan!(plan, :tftp)
|
|
97
|
+
if plan.allow_fixed_name_reuse && (!preserve_history || !verification_root)
|
|
98
|
+
raise ArgumentError, "fixed-name TFTP uploads require local history archival"
|
|
99
|
+
end
|
|
100
|
+
archive = TftpArchive.new(directory: report_directory, root: verification_root) if preserve_history
|
|
89
101
|
run_batch(:tftp, plan, concurrency: concurrency, report_directory: report_directory,
|
|
90
|
-
|
|
91
|
-
|
|
102
|
+
settings_snapshot: settings_snapshot, success_policy: success_policy, on_start: on_start, on_result: on_result,
|
|
103
|
+
verification: preserve_history ? nil : verification) do |device, log_directory|
|
|
104
|
+
if archive
|
|
105
|
+
archive.upload_and_archive(device, started_at: Time.now.utc) do |remote_filename|
|
|
106
|
+
tftp_backup_device(device, server, source_files, log_directory, vrfs, settings_snapshot, path: remote_filename)
|
|
107
|
+
end
|
|
108
|
+
else
|
|
109
|
+
tftp_backup_device(device, server, source_files, log_directory, vrfs, settings_snapshot)
|
|
110
|
+
end
|
|
92
111
|
end
|
|
93
112
|
end
|
|
94
113
|
|
|
95
114
|
private
|
|
96
115
|
|
|
97
116
|
# 策略在清单请求之前固定;注入的清单客户端仍由调用方负责其自身生命周期。
|
|
98
|
-
def inventory(
|
|
99
|
-
rows = (@client || @settings.client(policy:
|
|
117
|
+
def inventory(settings_snapshot)
|
|
118
|
+
rows = (@client || @settings.client(policy: settings_snapshot)).devices
|
|
100
119
|
raise Client::Error, "Netdisco returned an invalid device inventory" unless rows.is_a?(Array)
|
|
101
120
|
|
|
102
|
-
rules = @rules ||
|
|
121
|
+
rules = @rules || settings_snapshot.rules
|
|
103
122
|
devices = rows.map { |row| Device.from_row(row, rules: rules) }
|
|
104
123
|
duplicates = devices.group_by(&:host).reject { |host, entries| host.nil? || entries.size == 1 }
|
|
105
124
|
devices.map { |device| duplicates.key?(device.host) ? device.with(issue: :duplicate_host) : device }.freeze
|
|
106
125
|
end
|
|
107
126
|
|
|
108
|
-
def build_plan(mode, limit,
|
|
127
|
+
def build_plan(mode, limit, settings_snapshot, allow_fixed_name_reuse: false)
|
|
109
128
|
Planner.validate_limit!(limit)
|
|
110
|
-
Planner.new(inventory(
|
|
129
|
+
Planner.new(inventory(settings_snapshot)).call(mode: mode, limit_per_vendor: limit,
|
|
130
|
+
allow_fixed_name_reuse: allow_fixed_name_reuse)
|
|
111
131
|
end
|
|
112
132
|
|
|
113
133
|
# 统一组织设备任务、日志目录与报告;Worker 保证单台设备失败不影响其他设备。
|
|
114
|
-
def run_batch(mode, plan, concurrency:, report_directory:,
|
|
134
|
+
def run_batch(mode, plan, concurrency:, report_directory:, settings_snapshot:, success_policy:, on_start:, on_result:, output_directory: nil, verification: nil)
|
|
115
135
|
worker = Worker.new(concurrency: concurrency)
|
|
116
136
|
started_at = Time.now.utc
|
|
117
137
|
started = monotonic
|
|
118
138
|
outcomes = plan.outcomes.dup
|
|
119
139
|
FileUtils.mkdir_p(output_directory, mode: 0o700) if output_directory && !plan.ready.empty?
|
|
120
|
-
log_directory =
|
|
140
|
+
log_directory = settings_snapshot.log_directory
|
|
121
141
|
FileUtils.mkdir_p(log_directory, mode: 0o700) if log_directory && !plan.ready.empty?
|
|
122
142
|
callback_errors = worker.run(plan.ready, outcomes: outcomes, on_start: on_start, on_result: on_result,
|
|
123
143
|
on_error: ->(device, error) { build_outcome(device, :failed, error: error) }) do |device|
|
|
@@ -126,8 +146,9 @@ module Net
|
|
|
126
146
|
batch = Batch.new(mode: mode, outcomes: outcomes.freeze, started_at: started_at,
|
|
127
147
|
finished_at: Time.now.utc, callback_errors: callback_errors,
|
|
128
148
|
report_location: nil, report_error: nil)
|
|
129
|
-
|
|
130
|
-
|
|
149
|
+
report = Report.new(batch, policy: success_policy, duration_ms: ((monotonic - started) * 1000).round, log_directory: log_directory)
|
|
150
|
+
report = verification.call(report) if verification
|
|
151
|
+
save_report(report, directory: report_directory)
|
|
131
152
|
end
|
|
132
153
|
|
|
133
154
|
def monotonic = Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
|
@@ -138,16 +159,16 @@ module Net
|
|
|
138
159
|
end
|
|
139
160
|
|
|
140
161
|
# 保存批量报告,记录写入失败而不覆盖设备结果。
|
|
141
|
-
def save_report(
|
|
142
|
-
return
|
|
162
|
+
def save_report(report, directory:)
|
|
163
|
+
return report unless @result_store
|
|
143
164
|
|
|
144
|
-
|
|
165
|
+
report.with(report_location: @result_store.write(report, directory: directory))
|
|
145
166
|
rescue Storage::PrivateFile::WriteError => error
|
|
146
167
|
# 报告也可能已经原子替换;保留已知位置,不将持久性失败误写成完全没有产物。
|
|
147
168
|
location = error.receipt.path if Storage::PrivateFile.receipt_error?(error) && error.receipt.committed?
|
|
148
|
-
|
|
169
|
+
report.with_report_error(error, location: location)
|
|
149
170
|
rescue StandardError => error
|
|
150
|
-
|
|
171
|
+
report.with_report_error(error)
|
|
151
172
|
end
|
|
152
173
|
|
|
153
174
|
# 确认传入计划属于当前备份模式。
|
|
@@ -165,30 +186,30 @@ module Net
|
|
|
165
186
|
end
|
|
166
187
|
|
|
167
188
|
# 执行单台设备的 TFTP 导出。
|
|
168
|
-
def
|
|
169
|
-
|
|
170
|
-
|
|
189
|
+
def tftp_backup_device(device, server, source_files, log_directory, vrfs, settings_snapshot, path: device.tftp_filename)
|
|
190
|
+
run_device(device, log_directory, settings_snapshot, success_status: :reported_uploaded,
|
|
191
|
+
partial_status: :reported_with_error, backup_class: TftpReceipt) do |connector|
|
|
171
192
|
source_file = source_files.fetch(device.vendor, nil)
|
|
172
|
-
transfer_settings = { host: server, path:
|
|
193
|
+
transfer_settings = { host: server, path: path, source_file: source_file }
|
|
173
194
|
transfer_settings[:vrf] = vrfs.fetch(device.vendor) if vrfs.key?(device.vendor)
|
|
174
195
|
connector.tftp_backup(**transfer_settings)
|
|
175
196
|
end
|
|
176
197
|
end
|
|
177
198
|
|
|
178
199
|
# 执行单台设备的本地配置采集。
|
|
179
|
-
def
|
|
180
|
-
destination = File.join(directory, device.backup_filename)
|
|
200
|
+
def backup_device(device, directory, log_directory, settings_snapshot, filename_style: :ip)
|
|
201
|
+
destination = File.join(directory, device.backup_filename(style: filename_style))
|
|
181
202
|
Storage::BackupLock.synchronize(destination, host: device.host) do |path_lock|
|
|
182
203
|
Storage::SafeFile.open(destination, missing: true) { |_file, _stat| nil }
|
|
183
|
-
|
|
184
|
-
|
|
204
|
+
run_device(device, log_directory, settings_snapshot, success_status: :backed_up,
|
|
205
|
+
partial_status: :saved_with_error, backup_class: Backup) do |connector|
|
|
185
206
|
path_lock.with_delegated_lock { connector.backup(path: destination) }
|
|
186
207
|
end
|
|
187
208
|
end
|
|
188
209
|
end
|
|
189
210
|
|
|
190
211
|
# 隔离单台设备的凭据、连接、操作和关闭异常。
|
|
191
|
-
def
|
|
212
|
+
def run_device(device, log_directory, settings_snapshot, success_status:, partial_status:, backup_class:)
|
|
192
213
|
connection_settings = @credentials ? @credentials.call(device) : @settings.device_credentials_for(device)
|
|
193
214
|
return build_outcome(device, :missing_credentials) if connection_settings.nil?
|
|
194
215
|
unless connection_settings.is_a?(Hash) && connection_settings.keys.all?(Symbol) &&
|
|
@@ -197,7 +218,7 @@ module Net
|
|
|
197
218
|
raise ArgumentError, "credential resolver must return settings with username and without host"
|
|
198
219
|
end
|
|
199
220
|
|
|
200
|
-
connection_settings =
|
|
221
|
+
connection_settings = settings_snapshot.connection_options(device.vendor).merge(connection_settings)
|
|
201
222
|
log_name = device.host.tr(":", "_")
|
|
202
223
|
if log_directory
|
|
203
224
|
connection_settings = connection_settings.merge(log_file: File.join(log_directory, "#{log_name}.log"))
|
|
@@ -206,7 +227,7 @@ module Net
|
|
|
206
227
|
backup, failure = perform_one(connector, backup_class) { yield connector }
|
|
207
228
|
return build_outcome(device, success_status, backup: backup) unless failure
|
|
208
229
|
|
|
209
|
-
build_outcome(device, backup ?
|
|
230
|
+
build_outcome(device, backup ? partial_status : :failed, backup: backup, error: failure)
|
|
210
231
|
rescue StandardError => error
|
|
211
232
|
build_outcome(device, :failed, error: error)
|
|
212
233
|
end
|
|
@@ -23,21 +23,21 @@ module Net
|
|
|
23
23
|
# 在追加正文前计数;不能靠 Content-Length 或 JSON 解析后的大小限制内存。
|
|
24
24
|
def consume_bytes(size, response_bytes:)
|
|
25
25
|
remaining
|
|
26
|
-
|
|
27
|
-
|
|
26
|
+
validate_limit!(:max_response_bytes, response_bytes + size)
|
|
27
|
+
validate_limit!(:max_inventory_bytes, @bytes + size)
|
|
28
28
|
@bytes += size
|
|
29
29
|
end
|
|
30
30
|
|
|
31
31
|
# 去重前的记录同样占用清单内存;兼容查询也使用这个累计计数。
|
|
32
32
|
def consume_devices(size)
|
|
33
33
|
remaining
|
|
34
|
-
|
|
34
|
+
validate_limit!(:max_devices, @devices + size)
|
|
35
35
|
@devices += size
|
|
36
36
|
end
|
|
37
37
|
|
|
38
38
|
private
|
|
39
39
|
|
|
40
|
-
def
|
|
40
|
+
def validate_limit!(name, value)
|
|
41
41
|
return if value <= @options.fetch(name)
|
|
42
42
|
|
|
43
43
|
raise Client::Error.new("Netdisco inventory exceeded #{name}", code: name), cause: nil
|
|
@@ -4,15 +4,18 @@ module Net
|
|
|
4
4
|
module Connector
|
|
5
5
|
module Netdisco
|
|
6
6
|
# 固定一份清单上的选择结果与跳过原因,供预览和执行复用。
|
|
7
|
-
Plan = Data.define(:mode, :inventory, :ready, :outcomes) do
|
|
7
|
+
Plan = Data.define(:mode, :inventory, :ready, :outcomes, :allow_fixed_name_reuse) do
|
|
8
8
|
# Data 只冻结对象自身;这里复制并冻结可能由调用方修改的计划容器。
|
|
9
|
-
def initialize(mode:, inventory:, ready:, outcomes:)
|
|
9
|
+
def initialize(mode:, inventory:, ready:, outcomes:, allow_fixed_name_reuse: false)
|
|
10
10
|
unless inventory.is_a?(Array) && ready.is_a?(Array) && outcomes.is_a?(Array)
|
|
11
11
|
raise ArgumentError, "plan inventory, ready tasks, and outcomes must be Arrays"
|
|
12
12
|
end
|
|
13
13
|
|
|
14
14
|
tasks = ready.map { |task| task.is_a?(Array) ? task.dup.freeze : task }.freeze
|
|
15
|
-
|
|
15
|
+
raise ArgumentError, "allow_fixed_name_reuse must be boolean" unless [true, false].include?(allow_fixed_name_reuse)
|
|
16
|
+
|
|
17
|
+
super(mode: mode, inventory: inventory.dup.freeze, ready: tasks, outcomes: outcomes.dup.freeze,
|
|
18
|
+
allow_fixed_name_reuse: allow_fixed_name_reuse)
|
|
16
19
|
end
|
|
17
20
|
|
|
18
21
|
# 按清单顺序返回选中的设备。
|
|
@@ -26,7 +29,7 @@ module Net
|
|
|
26
29
|
end
|
|
27
30
|
|
|
28
31
|
selected_indices = {}
|
|
29
|
-
|
|
32
|
+
selected_filenames = {}
|
|
30
33
|
previous_index = -1
|
|
31
34
|
ready.each do |task|
|
|
32
35
|
unless task.is_a?(Array) && task.size == 2 && task.first.is_a?(Integer)
|
|
@@ -39,9 +42,11 @@ module Net
|
|
|
39
42
|
end
|
|
40
43
|
if mode == :tftp
|
|
41
44
|
filename = device.tftp_filename
|
|
42
|
-
|
|
45
|
+
if selected_filenames.key?(filename) && !(allow_fixed_name_reuse && device.vendor == :palo_alto)
|
|
46
|
+
raise ArgumentError, "plan cannot upload multiple devices to one TFTP filename"
|
|
47
|
+
end
|
|
43
48
|
|
|
44
|
-
|
|
49
|
+
selected_filenames[filename] = true
|
|
45
50
|
end
|
|
46
51
|
selected_indices[index] = true
|
|
47
52
|
previous_index = index
|
|
@@ -51,7 +56,7 @@ module Net
|
|
|
51
56
|
result = outcomes[index]
|
|
52
57
|
next if selected_indices[index] && result.nil?
|
|
53
58
|
|
|
54
|
-
unless !selected_indices[index] && valid_skipped_outcome?(result, device,
|
|
59
|
+
unless !selected_indices[index] && valid_skipped_outcome?(result, device, selected_filenames)
|
|
55
60
|
raise ArgumentError, "plan outcome does not match its inventory slot"
|
|
56
61
|
end
|
|
57
62
|
end
|
|
@@ -61,7 +66,7 @@ module Net
|
|
|
61
66
|
private
|
|
62
67
|
|
|
63
68
|
# 跳过结果不携带执行产物;文件冲突必须有本批次实际执行的目标作为依据。
|
|
64
|
-
def valid_skipped_outcome?(result, device,
|
|
69
|
+
def valid_skipped_outcome?(result, device, selected_filenames)
|
|
65
70
|
return false unless result.is_a?(Outcome) && result.device.equal?(device) && result.backup.nil? &&
|
|
66
71
|
result.error_code.nil? && result.error_type.nil? &&
|
|
67
72
|
result.started_at.nil? && result.finished_at.nil?
|
|
@@ -69,7 +74,7 @@ module Net
|
|
|
69
74
|
return result.status == device.issue unless device.ready?
|
|
70
75
|
|
|
71
76
|
result.status == :sample_limit ||
|
|
72
|
-
(mode == :tftp && result.status == :remote_filename_collision &&
|
|
77
|
+
(mode == :tftp && result.status == :remote_filename_collision && selected_filenames.key?(device.tftp_filename))
|
|
73
78
|
end
|
|
74
79
|
end
|
|
75
80
|
end
|