net-connector 0.5.0 → 0.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +58 -0
- data/CONTRIBUTING.md +20 -0
- data/README.md +153 -42
- data/docs/VERIFICATION.md +42 -1
- data/docs/architecture.md +20 -1
- data/examples/backup.yml +40 -0
- data/examples/inventory_sql.yml +16 -0
- data/lib/net/connector/device/base.rb +30 -38
- data/lib/net/connector/device/profile/builder.rb +8 -8
- data/lib/net/connector/device/profile.rb +2 -2
- data/lib/net/connector/device/running_config.rb +2 -2
- data/lib/net/connector/device/tftp/receipt.rb +12 -3
- data/lib/net/connector/device/tftp.rb +7 -7
- data/lib/net/connector/device/topology.rb +4 -4
- data/lib/net/connector/engine/authentication.rb +1 -1
- data/lib/net/connector/engine/command.rb +2 -2
- data/lib/net/connector/engine/configuration.rb +5 -2
- data/lib/net/connector/engine/dialogue.rb +4 -4
- data/lib/net/connector/engine/execution.rb +27 -16
- data/lib/net/connector/engine/known_hosts.rb +112 -0
- data/lib/net/connector/engine/log.rb +10 -7
- data/lib/net/connector/engine/session.rb +117 -128
- data/lib/net/connector/engine/transport.rb +32 -7
- data/lib/net/connector/netdisco/backup_run.rb +132 -0
- data/lib/net/connector/netdisco/batch.rb +1 -1
- data/lib/net/connector/netdisco/cli/options.rb +139 -0
- data/lib/net/connector/netdisco/cli.rb +20 -24
- data/lib/net/connector/netdisco/client.rb +8 -7
- data/lib/net/connector/netdisco/config_file.rb +26 -22
- data/lib/net/connector/netdisco/connection.rb +38 -0
- data/lib/net/connector/netdisco/database_client.rb +197 -0
- data/lib/net/connector/netdisco/device.rb +11 -1
- data/lib/net/connector/netdisco/fleet.rb +68 -47
- data/lib/net/connector/netdisco/inventory_budget.rb +4 -4
- data/lib/net/connector/netdisco/plan.rb +14 -9
- data/lib/net/connector/netdisco/planner.rb +18 -14
- data/lib/net/connector/netdisco/progress.rb +253 -0
- data/lib/net/connector/netdisco/report/files.rb +41 -0
- data/lib/net/connector/netdisco/report/text.rb +45 -0
- data/lib/net/connector/netdisco/report.rb +53 -31
- data/lib/net/connector/netdisco/result_store.rb +3 -1
- data/lib/net/connector/netdisco/rules.rb +10 -2
- data/lib/net/connector/netdisco/settings.rb +136 -63
- data/lib/net/connector/netdisco/tftp_archive.rb +175 -0
- data/lib/net/connector/netdisco/tftp_history.rb +16 -0
- data/lib/net/connector/netdisco/tftp_verification.rb +58 -0
- data/lib/net/connector/netdisco/worker.rb +8 -3
- data/lib/net/connector/netdisco.rb +11 -0
- data/lib/net/connector/storage/backup_lock.rb +1 -1
- data/lib/net/connector/storage/batch_directory.rb +29 -0
- data/lib/net/connector/storage.rb +1 -0
- data/lib/net/connector/vendor/h3c.rb +2 -1
- data/lib/net/connector/vendor/hillstone/tftp_backup.rb +3 -3
- data/lib/net/connector/vendor/palo_alto/tftp_backup.rb +3 -3
- data/lib/net/connector/vendor/radware/running_config.rb +19 -0
- data/lib/net/connector/vendor/radware.rb +4 -3
- data/lib/net/connector/version.rb +1 -1
- data/lib/net/connector.rb +2 -2
- metadata +57 -104
|
@@ -0,0 +1,139 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "optparse"
|
|
4
|
+
|
|
5
|
+
module Net
|
|
6
|
+
module Connector
|
|
7
|
+
module Netdisco
|
|
8
|
+
class CLI
|
|
9
|
+
# 可注入参数与输入输出的备份选项,不退出宿主进程。
|
|
10
|
+
class Options
|
|
11
|
+
def self.parse(argv: ARGV, input: $stdin, output: $stdout, error: $stderr, program: $PROGRAM_NAME)
|
|
12
|
+
values = { environment: {}, json: false, verbose: false, stdin_credentials: false }
|
|
13
|
+
parser = parser_for(values, output: output, program: program)
|
|
14
|
+
remaining = parser.parse!(argv.dup)
|
|
15
|
+
return values if values[:help]
|
|
16
|
+
raise OptionParser::InvalidArgument unless remaining.empty?
|
|
17
|
+
credentials = values[:environment].keys.any? { |key| key.match?(/USERNAME|PASSWORD/) }
|
|
18
|
+
if values[:stdin_credentials] && (credentials || values[:ask_password] || values[:ask_netdisco_password])
|
|
19
|
+
raise ArgumentError, "--stdin-credentials 不能与命令行凭据或密码提示同时使用"
|
|
20
|
+
end
|
|
21
|
+
read_password(values, :ask_password, "NC_DEVICE_PASSWORD", "设备密码", input: input, output: error)
|
|
22
|
+
read_password(values, :ask_netdisco_password, "NETDISCO_PASSWORD", "Netdisco 密码", input: input, output: error)
|
|
23
|
+
|
|
24
|
+
values
|
|
25
|
+
rescue OptionParser::ParseError
|
|
26
|
+
raise ArgumentError, "参数无效:并发数须为 1 至 50,抽样须为 1 至 5;使用 --help 查看参数", cause: nil
|
|
27
|
+
end
|
|
28
|
+
|
|
29
|
+
def self.parser_for(values, output:, program:)
|
|
30
|
+
OptionParser.new do |opts|
|
|
31
|
+
opts.banner = "用法:ruby #{File.basename(program)} [选项](默认全量)"
|
|
32
|
+
add_settings(opts, values)
|
|
33
|
+
add_execution(opts, values)
|
|
34
|
+
opts.on("--ask-password", "隐藏输入设备密码") { values[:ask_password] = true }
|
|
35
|
+
opts.on("--ask-netdisco-password", "隐藏输入 Netdisco 密码") { values[:ask_netdisco_password] = true }
|
|
36
|
+
opts.on("--verbose", "显示逐条登录和命令事件") { values[:verbose] = true }
|
|
37
|
+
opts.on("--json", "在 STDOUT 输出计划和结果 JSON") { values[:json] = true }
|
|
38
|
+
opts.on("--stdin-credentials", "从标准输入读取凭据并等待 RUN") { values[:stdin_credentials] = true }
|
|
39
|
+
opts.on("-h", "--help", "显示帮助") { output.puts opts; values[:help] = true }
|
|
40
|
+
end
|
|
41
|
+
end
|
|
42
|
+
|
|
43
|
+
def self.add_execution(opts, values)
|
|
44
|
+
opts.on("-c", "--concurrency N", Integer, "并发设备数,1 至 50") do |count|
|
|
45
|
+
raise OptionParser::InvalidArgument unless (1..Settings::MAX_CONCURRENCY).cover?(count)
|
|
46
|
+
|
|
47
|
+
values[:environment]["NC_CONCURRENCY"] = count.to_s
|
|
48
|
+
end
|
|
49
|
+
opts.on("--sample N", "--limit-per-vendor N", Integer, "每厂商抽样 1 至 5 台") do |count|
|
|
50
|
+
Planner.validate_limit!(count)
|
|
51
|
+
values[:limit] = count
|
|
52
|
+
end
|
|
53
|
+
opts.on("--tftp-root DIR", "本机 TFTP 服务器文件目录") { |path| values[:tftp_root] = path }
|
|
54
|
+
opts.on("--all", "选择全部就绪设备") { values[:all] = true }
|
|
55
|
+
opts.on("--success-policy POLICY", %w[strict selected verified], "成功判定:strict / selected / verified(TFTP 文件核验)") do |policy|
|
|
56
|
+
values[:success_policy] = policy.to_sym
|
|
57
|
+
end
|
|
58
|
+
end
|
|
59
|
+
|
|
60
|
+
def self.settings(options, env: ENV)
|
|
61
|
+
raise ArgumentError, "--all 不能与抽样同时使用" if options[:all] && options[:limit]
|
|
62
|
+
|
|
63
|
+
environment = environment(options, env: env)
|
|
64
|
+
path = environment["NC_CONFIG"]
|
|
65
|
+
defaults = path && !path.empty? ? ConfigFile.load(path) : {}
|
|
66
|
+
overrides = {}
|
|
67
|
+
{ source: "NETDISCO_SOURCE", query: "NETDISCO_QUERY", query_params: "NETDISCO_QUERY_PARAMS",
|
|
68
|
+
max_script_output_bytes: "NC_MAX_SCRIPT_OUTPUT_BYTES", host: "NC_INCLUDE_HOSTS",
|
|
69
|
+
limit: "NC_SAMPLE_PER_VENDOR" }.each do |option, key|
|
|
70
|
+
overrides[key] = options[option].to_s if options.key?(option)
|
|
71
|
+
end
|
|
72
|
+
overrides["NC_SAMPLE_PER_VENDOR"] = nil if options[:all]
|
|
73
|
+
Settings.new(env: environment, defaults: defaults, overrides: overrides)
|
|
74
|
+
end
|
|
75
|
+
|
|
76
|
+
def self.add_settings(opts, values)
|
|
77
|
+
{
|
|
78
|
+
"--username USER" => ["NC_DEVICE_USERNAME", "设备登录用户名"],
|
|
79
|
+
"--password PASSWORD" => ["NC_DEVICE_PASSWORD", "设备登录密码(可改用 --ask-password)"],
|
|
80
|
+
"--enable-password PASSWORD" => ["NC_ENABLE_PASSWORD", "设备提权密码"],
|
|
81
|
+
"--netdisco-url URL" => ["NETDISCO_URL", "Netdisco 地址"],
|
|
82
|
+
"--netdisco-username USER" => ["NETDISCO_USERNAME", "Netdisco 用户名"],
|
|
83
|
+
"--netdisco-password PASSWORD" => ["NETDISCO_PASSWORD", "Netdisco 密码"],
|
|
84
|
+
"--directory DIR" => ["NC_BACKUP_DIRECTORY", "备份目录,相对项目根目录"],
|
|
85
|
+
"--config FILE" => ["NC_CONFIG", "YAML 配置,相对项目根目录"]
|
|
86
|
+
}.each do |flag, (key, description)|
|
|
87
|
+
opts.on(flag, description) do |value|
|
|
88
|
+
raise OptionParser::InvalidArgument if value.empty?
|
|
89
|
+
|
|
90
|
+
values[:environment][key] = value
|
|
91
|
+
end
|
|
92
|
+
end
|
|
93
|
+
opts.on("--host-key-policy POLICY", %w[strict accept_new replace], "主机密钥策略:strict / accept_new / replace(变化时自动替换)") do |policy|
|
|
94
|
+
values[:environment]["NC_HOST_KEY_POLICY"] = policy
|
|
95
|
+
end
|
|
96
|
+
opts.on("--known-hosts FILE", "持久保存主机密钥的文件") do |path|
|
|
97
|
+
values[:environment]["NC_KNOWN_HOSTS"] = path
|
|
98
|
+
end
|
|
99
|
+
end
|
|
100
|
+
|
|
101
|
+
def self.environment(options, env: ENV)
|
|
102
|
+
result = env.to_h.dup
|
|
103
|
+
overrides = options.fetch(:environment)
|
|
104
|
+
# 显式设备参数覆盖对应厂商默认值;未指定的字段继续按原凭据规则解析。
|
|
105
|
+
{ "NC_DEVICE_USERNAME" => "USERNAME", "NC_DEVICE_PASSWORD" => "PASSWORD",
|
|
106
|
+
"NC_ENABLE_PASSWORD" => "ENABLE_PASSWORD" }.each do |key, suffix|
|
|
107
|
+
next unless overrides.key?(key)
|
|
108
|
+
|
|
109
|
+
result.delete_if do |name, _|
|
|
110
|
+
name.start_with?("NC_") && name.end_with?("_#{suffix}") &&
|
|
111
|
+
(suffix != "PASSWORD" || !name.end_with?("_ENABLE_PASSWORD"))
|
|
112
|
+
end
|
|
113
|
+
end
|
|
114
|
+
if overrides.key?("NETDISCO_USERNAME") || overrides.key?("NETDISCO_PASSWORD")
|
|
115
|
+
result.delete("NETDISCO_API_KEY")
|
|
116
|
+
end
|
|
117
|
+
result.merge(overrides)
|
|
118
|
+
end
|
|
119
|
+
|
|
120
|
+
def self.read_password(values, flag, key, label, input:, output:)
|
|
121
|
+
return unless values[flag]
|
|
122
|
+
|
|
123
|
+
raise ArgumentError, "密码提示与显式密码不能同时使用" if values[:environment].key?(key)
|
|
124
|
+
raise ArgumentError, "密码提示需要交互终端;自动化任务可使用环境变量或 --stdin-credentials" unless input.tty?
|
|
125
|
+
require "io/console"
|
|
126
|
+
output.print "#{label}:"
|
|
127
|
+
output.flush
|
|
128
|
+
password = input.noecho(&:gets)&.chomp
|
|
129
|
+
output.puts
|
|
130
|
+
raise ArgumentError, "密码不能为空" if password.nil? || password.empty?
|
|
131
|
+
|
|
132
|
+
values[:environment][key] = password
|
|
133
|
+
end
|
|
134
|
+
private_class_method :read_password, :parser_for
|
|
135
|
+
end
|
|
136
|
+
end
|
|
137
|
+
end
|
|
138
|
+
end
|
|
139
|
+
end
|
|
@@ -54,7 +54,10 @@ module Net
|
|
|
54
54
|
message = Storage::PrivateFile.receipt_error?(exception) ? exception.message : exception.class.name
|
|
55
55
|
@error.puts "net-connector-backup: #{message}"
|
|
56
56
|
2
|
|
57
|
-
rescue OptionParser::ParseError
|
|
57
|
+
rescue OptionParser::ParseError
|
|
58
|
+
@error.puts "net-connector-backup: 参数无效;使用 --help 查看参数"
|
|
59
|
+
2
|
|
60
|
+
rescue ArgumentError, KeyError, Errno::ENOENT,
|
|
58
61
|
Psych::Exception, JSON::ParserError, Client::Error => exception
|
|
59
62
|
@error.puts "net-connector-backup: #{exception.message}"
|
|
60
63
|
2
|
|
@@ -67,21 +70,13 @@ module Net
|
|
|
67
70
|
|
|
68
71
|
# 文件、环境变量、命令行按优先级覆盖,凭据仍只来自环境变量。
|
|
69
72
|
def settings_for(options)
|
|
70
|
-
|
|
71
|
-
values = {}
|
|
72
|
-
values["NET_CONNECTOR_BACKUP_DIRECTORY"] = options[:directory] if options[:directory]
|
|
73
|
-
values["NET_CONNECTOR_CONCURRENCY"] = options[:concurrency].to_s if options[:concurrency]
|
|
74
|
-
values["NET_CONNECTOR_MAX_SCRIPT_OUTPUT_BYTES"] = options[:max_script_output_bytes].to_s if options[:max_script_output_bytes]
|
|
75
|
-
values["NET_CONNECTOR_INCLUDE_HOSTS"] = options[:host] if options[:host]
|
|
76
|
-
values["NET_CONNECTOR_SAMPLE_PER_VENDOR"] = options[:limit].to_s if options[:limit]
|
|
77
|
-
values["NET_CONNECTOR_SAMPLE_PER_VENDOR"] = nil if options[:all]
|
|
78
|
-
Settings.new(env: @env, defaults: defaults, overrides: values)
|
|
73
|
+
Options.settings(options, env: @env)
|
|
79
74
|
end
|
|
80
75
|
|
|
81
76
|
# 导出已有配置不创建清单或设备连接。
|
|
82
77
|
def export_config(settings, options)
|
|
83
78
|
path = Storage::SavedConfig.new(directory: settings.backup_directory)
|
|
84
|
-
|
|
79
|
+
.export(host: options[:export], output: options[:output], io: @output)
|
|
85
80
|
@output.puts JSON.generate(host: options[:export], output: path) if path
|
|
86
81
|
end
|
|
87
82
|
|
|
@@ -107,7 +102,7 @@ module Net
|
|
|
107
102
|
fleet.tftp_backup_all(plan: plan, server: settings.tftp_server,
|
|
108
103
|
concurrency: settings.concurrency,
|
|
109
104
|
vrfs: settings.tftp_vrfs,
|
|
110
|
-
source_files: settings.tftp_source_files, **reporting)
|
|
105
|
+
source_files: settings.tftp_source_files, verification_root: options[:tftp_root] || @env["TFTP_ROOT"], **reporting)
|
|
111
106
|
else
|
|
112
107
|
fleet.backup_all(plan: plan, directory: settings.backup_directory,
|
|
113
108
|
concurrency: settings.concurrency, **reporting)
|
|
@@ -116,24 +111,21 @@ module Net
|
|
|
116
111
|
|
|
117
112
|
# 解析命令行参数并处理帮助及版本信息。
|
|
118
113
|
def parse_options
|
|
119
|
-
options = {
|
|
114
|
+
options = { environment: {} }
|
|
120
115
|
parser = OptionParser.new do |args|
|
|
121
116
|
args.banner = "用法:net-connector-backup [--config FILE] [--plan | --show-config | --export IP] [--tftp]"
|
|
122
|
-
|
|
117
|
+
Options.add_settings(args, options)
|
|
118
|
+
Options.add_execution(args, options)
|
|
119
|
+
args.on("--source SOURCE", %w[http postgres], "清单来源:http(默认)或 postgres") { |value| options[:source] = value }
|
|
120
|
+
args.on("--query SQL", "PostgreSQL 清单查询,使用 $1、$2 绑定参数") { |value| options[:query] = value }
|
|
121
|
+
args.on("--query-params JSON", "查询参数的 JSON 数组") { |value| options[:query_params] = value }
|
|
123
122
|
args.on("--show-config", "显示生效的非敏感设置") { options[:show_config] = true }
|
|
124
123
|
args.on("--plan", "预览设备清单,不连接设备") { options[:plan] = true }
|
|
125
124
|
args.on("--tftp", "由设备发起 TFTP 备份") { options[:tftp] = true }
|
|
126
125
|
args.on("--export IP", "导出已保存的本地配置") { |value| options[:export] = value }
|
|
127
126
|
args.on("--output FILE", "将导出配置写入私有文件") { |value| options[:output] = value }
|
|
128
|
-
args.on("--directory PATH", "备份和报告目录") { |value| options[:directory] = value }
|
|
129
127
|
args.on("--host IP", "按管理地址选择一台设备") { |value| options[:host] = value }
|
|
130
|
-
args.on("--concurrency N", Integer, "最大并发设备数") { |value| options[:concurrency] = value }
|
|
131
128
|
args.on("--max-script-output-bytes N", Integer, "每个脚本的累计响应字节上限") { |value| options[:max_script_output_bytes] = value }
|
|
132
|
-
args.on("--limit-per-vendor N", Integer, "每厂商选择 N 台设备(1 至 5)") do |value|
|
|
133
|
-
options[:limit] = value
|
|
134
|
-
end
|
|
135
|
-
args.on("--all", "选择全部就绪设备") { options[:all] = true }
|
|
136
|
-
args.on("--success-policy POLICY", %w[strict selected], "成功判定:strict(默认)或 selected") { |value| options[:success_policy] = value.to_sym }
|
|
137
129
|
args.on("-v", "--version", "显示版本") do
|
|
138
130
|
@output.puts Net::Connector::VERSION
|
|
139
131
|
options[:done] = true
|
|
@@ -151,15 +143,19 @@ module Net
|
|
|
151
143
|
|
|
152
144
|
# 拒绝互斥或缺少依赖的命令行选项。
|
|
153
145
|
def validate_options!(options)
|
|
146
|
+
raise ArgumentError, "verified policy requires --tftp" if options[:success_policy] == :verified && !options[:tftp]
|
|
154
147
|
Report.validate_policy!(options.fetch(:success_policy, :strict))
|
|
155
148
|
Planner.validate_limit!(options[:limit])
|
|
156
|
-
|
|
149
|
+
raise ArgumentError, "--tftp-root requires --tftp" if options[:tftp_root] && !options[:tftp]
|
|
157
150
|
modes = [:show_config, :plan, :export].count { |key| options[key] }
|
|
158
151
|
raise ArgumentError, "--show-config、--plan 和 --export 只能选择一项" if modes > 1
|
|
159
152
|
raise ArgumentError, "--output 需要同时指定 --export" if options[:output] && !options[:export]
|
|
160
153
|
raise ArgumentError, "--all 不能与 --limit-per-vendor 同时使用" if options[:all] && options[:limit]
|
|
161
|
-
if options[:export] && (options[:
|
|
162
|
-
|
|
154
|
+
if options[:export] && (options[:source] || options[:query] || options[:query_params])
|
|
155
|
+
raise ArgumentError, "--export 不能与清单查询选项同时使用"
|
|
156
|
+
end
|
|
157
|
+
if options[:export] && (options[:tftp] || options[:all] || options[:limit] || options[:environment].key?("NC_CONCURRENCY") || options[:host] ||
|
|
158
|
+
options[:success_policy] || options[:tftp_root] || options[:max_script_output_bytes])
|
|
163
159
|
raise ArgumentError, "--export 不能与备份选项同时使用"
|
|
164
160
|
end
|
|
165
161
|
end
|
|
@@ -96,7 +96,7 @@ module Net
|
|
|
96
96
|
@options.fetch(:max_pages).times do |index|
|
|
97
97
|
uri = endpoint("api/v1/search/device")
|
|
98
98
|
uri.query = URI.encode_www_form(fields: FIELDS.join(","), limit: @options.fetch(:page_size),
|
|
99
|
-
|
|
99
|
+
offset: index * @options.fetch(:page_size))
|
|
100
100
|
page = request_json(uri, authorized_get(uri, token), budget, allow_empty: true, query_fallback: true)
|
|
101
101
|
validate_rows!(page, budget)
|
|
102
102
|
return rows.uniq if page.empty?
|
|
@@ -258,9 +258,9 @@ module Net
|
|
|
258
258
|
row.is_a?(Hash) && present?(row["ip"]) &&
|
|
259
259
|
FIELDS.drop(1).all? { |field| row[field].nil? || row[field].is_a?(String) }
|
|
260
260
|
end
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
261
|
+
return if valid
|
|
262
|
+
|
|
263
|
+
raise Error.new("Netdisco returned an invalid device inventory", code: :invalid_inventory), cause: nil
|
|
264
264
|
end
|
|
265
265
|
|
|
266
266
|
def validate_credentials!(username, password, api_key)
|
|
@@ -268,12 +268,13 @@ module Net
|
|
|
268
268
|
(present?(api_key) && username.nil? && password.nil?)
|
|
269
269
|
raise ArgumentError, "provide either username and password or api_key"
|
|
270
270
|
end
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
271
|
+
return unless [username, password, api_key].compact.any? { |value| value.match?(/[\r\n\x00]/) }
|
|
272
|
+
|
|
273
|
+
raise ArgumentError, "credentials must be single-line strings"
|
|
274
274
|
end
|
|
275
275
|
|
|
276
276
|
def monotonic = Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
|
277
|
+
|
|
277
278
|
def present?(value) = value.is_a?(String) && !value.strip.empty?
|
|
278
279
|
end
|
|
279
280
|
end
|
|
@@ -9,7 +9,10 @@ module Net
|
|
|
9
9
|
# 命令行使用的无凭据 YAML 配置,环境变量优先。
|
|
10
10
|
class ConfigFile
|
|
11
11
|
FIELDS = {
|
|
12
|
-
"netdisco" => { "
|
|
12
|
+
"netdisco" => { "source" => ["NETDISCO_SOURCE", :string],
|
|
13
|
+
"query" => ["NETDISCO_QUERY", :string],
|
|
14
|
+
"query_params" => ["NETDISCO_QUERY_PARAMS", :json_array],
|
|
15
|
+
"url" => ["NETDISCO_URL", :string],
|
|
13
16
|
"page_size" => ["NETDISCO_PAGE_SIZE", :integer],
|
|
14
17
|
"max_pages" => ["NETDISCO_MAX_PAGES", :integer],
|
|
15
18
|
"max_response_bytes" => ["NETDISCO_MAX_RESPONSE_BYTES", :integer],
|
|
@@ -17,26 +20,27 @@ module Net
|
|
|
17
20
|
"max_devices" => ["NETDISCO_MAX_DEVICES", :integer],
|
|
18
21
|
"inventory_timeout" => ["NETDISCO_INVENTORY_TIMEOUT", :number],
|
|
19
22
|
"allow_insecure_http" => ["NETDISCO_ALLOW_INSECURE_HTTP", :boolean] },
|
|
20
|
-
"backup" => { "directory" => ["
|
|
21
|
-
"concurrency" => ["
|
|
22
|
-
"limit_per_vendor" => ["
|
|
23
|
-
"inventory" => { "include_hosts" => ["
|
|
24
|
-
"exclude_hosts" => ["
|
|
25
|
-
"include_vendors" => ["
|
|
26
|
-
"vendor_overrides" => ["
|
|
27
|
-
"host_overrides" => ["
|
|
28
|
-
"device_rules" => ["
|
|
29
|
-
"ssh" => { "protocol" => ["
|
|
30
|
-
"
|
|
31
|
-
"
|
|
32
|
-
"
|
|
33
|
-
"
|
|
34
|
-
"
|
|
23
|
+
"backup" => { "directory" => ["NC_BACKUP_DIRECTORY", :string],
|
|
24
|
+
"concurrency" => ["NC_CONCURRENCY", :integer],
|
|
25
|
+
"limit_per_vendor" => ["NC_SAMPLE_PER_VENDOR", :integer] },
|
|
26
|
+
"inventory" => { "include_hosts" => ["NC_INCLUDE_HOSTS", :list],
|
|
27
|
+
"exclude_hosts" => ["NC_EXCLUDE_HOSTS", :list],
|
|
28
|
+
"include_vendors" => ["NC_INCLUDE_VENDORS", :list],
|
|
29
|
+
"vendor_overrides" => ["NC_VENDOR_OVERRIDES", :json],
|
|
30
|
+
"host_overrides" => ["NC_HOST_OVERRIDES", :json],
|
|
31
|
+
"device_rules" => ["NC_DEVICE_RULES", :json] },
|
|
32
|
+
"ssh" => { "protocol" => ["NC_PROTOCOL", :string],
|
|
33
|
+
"vendor_protocols" => ["NC_VENDOR_PROTOCOLS", :json],
|
|
34
|
+
"max_script_output_bytes" => ["NC_MAX_SCRIPT_OUTPUT_BYTES", :optional_integer],
|
|
35
|
+
"known_hosts" => ["NC_KNOWN_HOSTS", :string],
|
|
36
|
+
"host_key_policy" => ["NC_HOST_KEY_POLICY", :string],
|
|
37
|
+
"log_directory" => ["NC_LOG_DIRECTORY", :string],
|
|
38
|
+
"log_level" => ["NC_LOG_LEVEL", :string] },
|
|
35
39
|
"tftp" => { "server" => ["TFTP_HOST", :string],
|
|
36
|
-
"vrfs" => ["
|
|
37
|
-
"h3c_source_file" => ["
|
|
38
|
-
"h3c_wireless_source_file" => ["
|
|
39
|
-
"huawei_source_file" => ["
|
|
40
|
+
"vrfs" => ["NC_TFTP_VRFS", :json],
|
|
41
|
+
"h3c_source_file" => ["NC_H3C_TFTP_SOURCE_FILE", :string],
|
|
42
|
+
"h3c_wireless_source_file" => ["NC_H3C_WIRELESS_TFTP_SOURCE_FILE", :string],
|
|
43
|
+
"huawei_source_file" => ["NC_HUAWEI_TFTP_SOURCE_FILE", :string] }
|
|
40
44
|
}.freeze
|
|
41
45
|
|
|
42
46
|
# 安全读取 YAML 配置并映射到环境变量格式。
|
|
@@ -94,8 +98,8 @@ module Net
|
|
|
94
98
|
end
|
|
95
99
|
|
|
96
100
|
value.join(",")
|
|
97
|
-
when :json
|
|
98
|
-
expected = name == "device_rules" ? Array : Hash
|
|
101
|
+
when :json, :json_array
|
|
102
|
+
expected = type == :json_array || name == "device_rules" ? Array : Hash
|
|
99
103
|
raise ArgumentError, "#{section}.#{name} must be a #{expected}" unless value.is_a?(expected)
|
|
100
104
|
|
|
101
105
|
JSON.generate(value)
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "json"
|
|
4
|
+
require_relative "client"
|
|
5
|
+
|
|
6
|
+
# 清单连接与设备凭据来源;业务入口负责规划和执行备份。
|
|
7
|
+
module Net
|
|
8
|
+
module Connector
|
|
9
|
+
module Netdisco
|
|
10
|
+
# 将调用方提供的输入解析为清单客户端和设备凭据 resolver。
|
|
11
|
+
class Connection
|
|
12
|
+
def self.build(settings, stdin_credentials: false, input: $stdin)
|
|
13
|
+
unless stdin_credentials
|
|
14
|
+
return [settings.client, settings.method(:device_credentials_for)]
|
|
15
|
+
end
|
|
16
|
+
|
|
17
|
+
begin
|
|
18
|
+
values = JSON.parse(input.gets || "")
|
|
19
|
+
rescue JSON::ParserError
|
|
20
|
+
raise ArgumentError, "凭据 JSON 无效", cause: nil
|
|
21
|
+
end
|
|
22
|
+
keys = %w[netdisco_username netdisco_password device_username device_password]
|
|
23
|
+
raise ArgumentError, "需要四个非空凭据字符串" unless values.is_a?(Hash) && keys.all? { |key| values[key].is_a?(String) && !values[key].strip.empty? }
|
|
24
|
+
|
|
25
|
+
client = settings.client(credentials: {
|
|
26
|
+
username: values.fetch("netdisco_username"), password: values.fetch("netdisco_password")
|
|
27
|
+
})
|
|
28
|
+
credentials = lambda do |device|
|
|
29
|
+
inherited = device ? settings.device_credentials_for(device) : nil
|
|
30
|
+
(inherited || {}).merge(username: values.fetch("device_username"),
|
|
31
|
+
password: values.fetch("device_password"))
|
|
32
|
+
end
|
|
33
|
+
[client, credentials]
|
|
34
|
+
end
|
|
35
|
+
end
|
|
36
|
+
end
|
|
37
|
+
end
|
|
38
|
+
end
|
|
@@ -0,0 +1,197 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "timeout"
|
|
4
|
+
require_relative "client"
|
|
5
|
+
|
|
6
|
+
module Net
|
|
7
|
+
module Connector
|
|
8
|
+
module Netdisco
|
|
9
|
+
# SQL 负责业务选择和列别名;适配器只管理只读查询、清单预算与连接生命周期。
|
|
10
|
+
class DatabaseClient
|
|
11
|
+
CONNECTION_ENV = {
|
|
12
|
+
host: "NETDISCO_DB_HOST", port: "NETDISCO_DB_PORT", dbname: "NETDISCO_DB_NAME",
|
|
13
|
+
user: "NETDISCO_DB_USER", password: "NETDISCO_DB_PASS",
|
|
14
|
+
sslmode: "NETDISCO_DB_SSLMODE", sslrootcert: "NETDISCO_DB_SSLROOTCERT",
|
|
15
|
+
connect_timeout: "NETDISCO_DB_CONNECT_TIMEOUT"
|
|
16
|
+
}.freeze
|
|
17
|
+
REQUIRED_CONNECTION_KEYS = %i[host dbname user password].freeze
|
|
18
|
+
DEFAULTS = Client::DEFAULTS.except(:allow_insecure_http).freeze
|
|
19
|
+
|
|
20
|
+
def self.options(query:, query_params: [], **values)
|
|
21
|
+
unless query.is_a?(String) && !query.strip.empty? && !query.include?("\0")
|
|
22
|
+
raise ArgumentError, "query must be a nonempty SQL string without NUL"
|
|
23
|
+
end
|
|
24
|
+
unless query_params.is_a?(Array) && query_params.all? { |value|
|
|
25
|
+
value.nil? || value == true || value == false || value.is_a?(Integer) ||
|
|
26
|
+
(value.is_a?(Float) && value.finite?) || (value.is_a?(String) && !value.include?("\0"))
|
|
27
|
+
}
|
|
28
|
+
raise ArgumentError, "query_params must be an array of JSON scalar values without NUL"
|
|
29
|
+
end
|
|
30
|
+
raise ArgumentError, "unknown database inventory options" unless (values.keys - DEFAULTS.keys).empty?
|
|
31
|
+
|
|
32
|
+
Client.options(**values).slice(*DEFAULTS.keys).merge(
|
|
33
|
+
query: query.dup.freeze,
|
|
34
|
+
query_params: query_params.map { |value| value.is_a?(String) ? value.dup.freeze : value }.freeze
|
|
35
|
+
).freeze
|
|
36
|
+
end
|
|
37
|
+
|
|
38
|
+
def initialize(connection_options:, **options)
|
|
39
|
+
@options = self.class.options(**options)
|
|
40
|
+
@connection_options = validate_connection_options!(connection_options)
|
|
41
|
+
end
|
|
42
|
+
|
|
43
|
+
# 每次查询独占连接和游标;只有全部结果验证成功后才把清单交给 Fleet。
|
|
44
|
+
def devices
|
|
45
|
+
require "pg"
|
|
46
|
+
budget = InventoryBudget.new(@options, clock: method(:monotonic))
|
|
47
|
+
connection = nil
|
|
48
|
+
connected = false
|
|
49
|
+
begin
|
|
50
|
+
Timeout.timeout(budget.remaining, Client::InventoryTimeout) do
|
|
51
|
+
# 从握手开始持有连接;PG.connect 尚未返回时被超时中断,调用方无法负责关闭它。
|
|
52
|
+
connection = PG::Connection.connect_start(@connection_options)
|
|
53
|
+
# PostgreSQL NOTICE 可能包含 SQL 或数据,不交给 libpq 默认的 stderr 输出器。
|
|
54
|
+
connection.set_notice_processor { |_notice| nil }
|
|
55
|
+
wait_for_connection(connection, budget)
|
|
56
|
+
connected = true
|
|
57
|
+
# 使用 pg 可中断的 Ruby 等待接口;底层 socket 仍以非阻塞方式工作。
|
|
58
|
+
connection.setnonblocking(false)
|
|
59
|
+
connection.set_client_encoding("UTF8")
|
|
60
|
+
connection.exec("BEGIN READ ONLY").clear
|
|
61
|
+
set_statement_timeout(connection, budget)
|
|
62
|
+
# 即使没有参数也走扩展查询协议,由 PostgreSQL 拒绝多语句和非查询命令。
|
|
63
|
+
connection.exec_params("DECLARE net_connector_inventory NO SCROLL CURSOR FOR\n#{@options.fetch(:query)}",
|
|
64
|
+
@options.fetch(:query_params)).clear
|
|
65
|
+
rows = inventory(connection, budget)
|
|
66
|
+
connection.exec("ROLLBACK").clear
|
|
67
|
+
budget.remaining
|
|
68
|
+
rows
|
|
69
|
+
end
|
|
70
|
+
rescue Client::Error => error
|
|
71
|
+
raise error, cause: nil
|
|
72
|
+
rescue PG::QueryCanceled
|
|
73
|
+
raise Client::InventoryTimeout, cause: nil
|
|
74
|
+
rescue PG::Error, IOError, SystemCallError
|
|
75
|
+
code = connected ? :query_failed : :connection_failed
|
|
76
|
+
raise Client::Error.new("Netdisco database #{code.to_s.tr("_", " ")}", code: code), cause: nil
|
|
77
|
+
ensure
|
|
78
|
+
connection&.finish
|
|
79
|
+
end
|
|
80
|
+
end
|
|
81
|
+
|
|
82
|
+
def inspect = "#<#{self.class}>"
|
|
83
|
+
|
|
84
|
+
private
|
|
85
|
+
|
|
86
|
+
def wait_for_connection(connection, budget)
|
|
87
|
+
deadline = monotonic + @connection_options.fetch(:connect_timeout, budget.remaining).to_f
|
|
88
|
+
loop do
|
|
89
|
+
status = connection.connect_poll
|
|
90
|
+
return if status == PG::PGRES_POLLING_OK
|
|
91
|
+
raise PG::ConnectionBad unless [PG::PGRES_POLLING_READING, PG::PGRES_POLLING_WRITING].include?(status)
|
|
92
|
+
|
|
93
|
+
remaining = [budget.remaining, deadline - monotonic].min
|
|
94
|
+
raise PG::ConnectionBad if remaining <= 0
|
|
95
|
+
|
|
96
|
+
socket = connection.socket_io
|
|
97
|
+
readers = status == PG::PGRES_POLLING_READING ? [socket] : nil
|
|
98
|
+
writers = status == PG::PGRES_POLLING_WRITING ? [socket] : nil
|
|
99
|
+
next if IO.select(readers, writers, [socket], remaining)
|
|
100
|
+
|
|
101
|
+
budget.remaining
|
|
102
|
+
raise PG::ConnectionBad
|
|
103
|
+
end
|
|
104
|
+
end
|
|
105
|
+
|
|
106
|
+
def validate_connection_options!(options)
|
|
107
|
+
unless options.is_a?(Hash) && (options.keys - CONNECTION_ENV.keys).empty?
|
|
108
|
+
raise ArgumentError, "connection_options must contain supported PostgreSQL connection keys"
|
|
109
|
+
end
|
|
110
|
+
REQUIRED_CONNECTION_KEYS.each do |key|
|
|
111
|
+
value = options[key]
|
|
112
|
+
unless value.is_a?(String) && !value.strip.empty?
|
|
113
|
+
raise ArgumentError, "#{CONNECTION_ENV.fetch(key)} is required"
|
|
114
|
+
end
|
|
115
|
+
end
|
|
116
|
+
options.each do |key, value|
|
|
117
|
+
valid = value.is_a?(String) && !value.empty? && !value.include?("\0")
|
|
118
|
+
valid ||= %i[port connect_timeout].include?(key) && value.is_a?(Integer)
|
|
119
|
+
raise ArgumentError, "#{CONNECTION_ENV.fetch(key)} is invalid" unless valid
|
|
120
|
+
end
|
|
121
|
+
%i[port connect_timeout].each do |key|
|
|
122
|
+
next unless options.key?(key)
|
|
123
|
+
|
|
124
|
+
value = options.fetch(key).to_s
|
|
125
|
+
maximum = key == :port ? 65_535 : 2_147_483_647
|
|
126
|
+
unless value.match?(/\A\d+\z/) && value.to_i.between?(1, maximum)
|
|
127
|
+
raise ArgumentError, "#{CONNECTION_ENV.fetch(key)} must be a positive integer within range"
|
|
128
|
+
end
|
|
129
|
+
end
|
|
130
|
+
if options[:sslmode] && !%w[disable allow prefer require verify-ca verify-full].include?(options[:sslmode])
|
|
131
|
+
raise ArgumentError, "NETDISCO_DB_SSLMODE is invalid"
|
|
132
|
+
end
|
|
133
|
+
# libpq 会把 dbname 中的 URI/conninfo 再解释为连接参数,不能借此绕过凭据来源。
|
|
134
|
+
if options.fetch(:dbname).match?(/=|\Apostgres(?:ql)?:\/\//)
|
|
135
|
+
raise ArgumentError, "NETDISCO_DB_NAME must be a database name, not a connection string"
|
|
136
|
+
end
|
|
137
|
+
options.transform_values { |value| value.is_a?(String) ? value.dup.freeze : value }.freeze
|
|
138
|
+
end
|
|
139
|
+
|
|
140
|
+
def set_statement_timeout(connection, budget)
|
|
141
|
+
milliseconds = [(budget.remaining * 1000).ceil, 2_147_483_647].min
|
|
142
|
+
connection.exec_params("SELECT set_config('statement_timeout', $1, true)", [milliseconds.to_s]).clear
|
|
143
|
+
end
|
|
144
|
+
|
|
145
|
+
def inventory(connection, budget)
|
|
146
|
+
rows = []
|
|
147
|
+
@options.fetch(:max_pages).times do
|
|
148
|
+
set_statement_timeout(connection, budget)
|
|
149
|
+
count = fetch_page(connection, rows, budget)
|
|
150
|
+
return rows if count < @options.fetch(:page_size)
|
|
151
|
+
end
|
|
152
|
+
raise Client::Error.new("Netdisco inventory exceeded max_pages", code: :max_pages), cause: nil
|
|
153
|
+
end
|
|
154
|
+
|
|
155
|
+
# 单行模式避免 libpq 先缓存整页;Ruby 只积累已经通过预算检查的白名单字段。
|
|
156
|
+
def fetch_page(connection, rows, budget)
|
|
157
|
+
connection.send_query_params("FETCH FORWARD #{@options.fetch(:page_size)} FROM net_connector_inventory", [])
|
|
158
|
+
connection.set_single_row_mode
|
|
159
|
+
count = 0
|
|
160
|
+
response_bytes = 0
|
|
161
|
+
while (result = connection.get_result)
|
|
162
|
+
begin
|
|
163
|
+
result.check
|
|
164
|
+
response_bytes = append_rows(result, rows, budget, response_bytes)
|
|
165
|
+
count += result.ntuples
|
|
166
|
+
ensure
|
|
167
|
+
result.clear
|
|
168
|
+
end
|
|
169
|
+
end
|
|
170
|
+
count
|
|
171
|
+
end
|
|
172
|
+
|
|
173
|
+
def append_rows(result, rows, budget, response_bytes)
|
|
174
|
+
fields = result.fields
|
|
175
|
+
unless fields.include?("ip") && fields.uniq == fields
|
|
176
|
+
raise Client::Error.new("Netdisco database query must return unique column names including ip",
|
|
177
|
+
code: :invalid_inventory), cause: nil
|
|
178
|
+
end
|
|
179
|
+
budget.consume_devices(result.ntuples)
|
|
180
|
+
result.each do |row|
|
|
181
|
+
# 未选用的列也占传输预算,但不会进入清单、计划或报告。
|
|
182
|
+
size = row.sum { |key, value| key.bytesize + (value&.bytesize || 0) }
|
|
183
|
+
budget.consume_bytes(size, response_bytes: response_bytes)
|
|
184
|
+
response_bytes += size
|
|
185
|
+
unless row["ip"].is_a?(String) && !row["ip"].strip.empty?
|
|
186
|
+
raise Client::Error.new("Netdisco returned an invalid device inventory", code: :invalid_inventory), cause: nil
|
|
187
|
+
end
|
|
188
|
+
rows << row.slice(*Client::FIELDS)
|
|
189
|
+
end
|
|
190
|
+
response_bytes
|
|
191
|
+
end
|
|
192
|
+
|
|
193
|
+
def monotonic = Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
|
194
|
+
end
|
|
195
|
+
end
|
|
196
|
+
end
|
|
197
|
+
end
|
|
@@ -44,7 +44,17 @@ module Net
|
|
|
44
44
|
def ready? = issue.nil?
|
|
45
45
|
|
|
46
46
|
# 生成不随名称或厂商信息改变的本地文本文件名。
|
|
47
|
-
def backup_filename
|
|
47
|
+
def backup_filename(style: :ip)
|
|
48
|
+
address = Storage::SavedConfig.filename(host)
|
|
49
|
+
return address if style == :ip
|
|
50
|
+
raise ArgumentError, "filename style must be ip or hostname_ip" unless style == :hostname_ip
|
|
51
|
+
|
|
52
|
+
label = name.to_s.encode("UTF-8", invalid: :replace, undef: :replace, replace: "-")
|
|
53
|
+
.gsub(/[^\p{L}\p{N}._-]+/, "-").byteslice(0, 160).scrub("")
|
|
54
|
+
.gsub(/\A[._-]+|[._-]+\z/, "")
|
|
55
|
+
label = "unnamed" if label.empty?
|
|
56
|
+
"#{label}-#{address}"
|
|
57
|
+
end
|
|
48
58
|
|
|
49
59
|
# 生成设备 CLI 可用的 ASCII 远端文件名。
|
|
50
60
|
def tftp_filename
|