net-connector 0.5.0 → 0.7.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (60) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +58 -0
  3. data/CONTRIBUTING.md +20 -0
  4. data/README.md +153 -42
  5. data/docs/VERIFICATION.md +42 -1
  6. data/docs/architecture.md +20 -1
  7. data/examples/backup.yml +40 -0
  8. data/examples/inventory_sql.yml +16 -0
  9. data/lib/net/connector/device/base.rb +30 -38
  10. data/lib/net/connector/device/profile/builder.rb +8 -8
  11. data/lib/net/connector/device/profile.rb +2 -2
  12. data/lib/net/connector/device/running_config.rb +2 -2
  13. data/lib/net/connector/device/tftp/receipt.rb +12 -3
  14. data/lib/net/connector/device/tftp.rb +7 -7
  15. data/lib/net/connector/device/topology.rb +4 -4
  16. data/lib/net/connector/engine/authentication.rb +1 -1
  17. data/lib/net/connector/engine/command.rb +2 -2
  18. data/lib/net/connector/engine/configuration.rb +5 -2
  19. data/lib/net/connector/engine/dialogue.rb +4 -4
  20. data/lib/net/connector/engine/execution.rb +27 -16
  21. data/lib/net/connector/engine/known_hosts.rb +112 -0
  22. data/lib/net/connector/engine/log.rb +10 -7
  23. data/lib/net/connector/engine/session.rb +117 -128
  24. data/lib/net/connector/engine/transport.rb +32 -7
  25. data/lib/net/connector/netdisco/backup_run.rb +132 -0
  26. data/lib/net/connector/netdisco/batch.rb +1 -1
  27. data/lib/net/connector/netdisco/cli/options.rb +139 -0
  28. data/lib/net/connector/netdisco/cli.rb +20 -24
  29. data/lib/net/connector/netdisco/client.rb +8 -7
  30. data/lib/net/connector/netdisco/config_file.rb +26 -22
  31. data/lib/net/connector/netdisco/connection.rb +38 -0
  32. data/lib/net/connector/netdisco/database_client.rb +197 -0
  33. data/lib/net/connector/netdisco/device.rb +11 -1
  34. data/lib/net/connector/netdisco/fleet.rb +68 -47
  35. data/lib/net/connector/netdisco/inventory_budget.rb +4 -4
  36. data/lib/net/connector/netdisco/plan.rb +14 -9
  37. data/lib/net/connector/netdisco/planner.rb +18 -14
  38. data/lib/net/connector/netdisco/progress.rb +253 -0
  39. data/lib/net/connector/netdisco/report/files.rb +41 -0
  40. data/lib/net/connector/netdisco/report/text.rb +45 -0
  41. data/lib/net/connector/netdisco/report.rb +53 -31
  42. data/lib/net/connector/netdisco/result_store.rb +3 -1
  43. data/lib/net/connector/netdisco/rules.rb +10 -2
  44. data/lib/net/connector/netdisco/settings.rb +136 -63
  45. data/lib/net/connector/netdisco/tftp_archive.rb +175 -0
  46. data/lib/net/connector/netdisco/tftp_history.rb +16 -0
  47. data/lib/net/connector/netdisco/tftp_verification.rb +58 -0
  48. data/lib/net/connector/netdisco/worker.rb +8 -3
  49. data/lib/net/connector/netdisco.rb +11 -0
  50. data/lib/net/connector/storage/backup_lock.rb +1 -1
  51. data/lib/net/connector/storage/batch_directory.rb +29 -0
  52. data/lib/net/connector/storage.rb +1 -0
  53. data/lib/net/connector/vendor/h3c.rb +2 -1
  54. data/lib/net/connector/vendor/hillstone/tftp_backup.rb +3 -3
  55. data/lib/net/connector/vendor/palo_alto/tftp_backup.rb +3 -3
  56. data/lib/net/connector/vendor/radware/running_config.rb +19 -0
  57. data/lib/net/connector/vendor/radware.rb +4 -3
  58. data/lib/net/connector/version.rb +1 -1
  59. data/lib/net/connector.rb +2 -2
  60. metadata +57 -104
@@ -0,0 +1,139 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "optparse"
4
+
5
+ module Net
6
+ module Connector
7
+ module Netdisco
8
+ class CLI
9
+ # 可注入参数与输入输出的备份选项,不退出宿主进程。
10
+ class Options
11
+ def self.parse(argv: ARGV, input: $stdin, output: $stdout, error: $stderr, program: $PROGRAM_NAME)
12
+ values = { environment: {}, json: false, verbose: false, stdin_credentials: false }
13
+ parser = parser_for(values, output: output, program: program)
14
+ remaining = parser.parse!(argv.dup)
15
+ return values if values[:help]
16
+ raise OptionParser::InvalidArgument unless remaining.empty?
17
+ credentials = values[:environment].keys.any? { |key| key.match?(/USERNAME|PASSWORD/) }
18
+ if values[:stdin_credentials] && (credentials || values[:ask_password] || values[:ask_netdisco_password])
19
+ raise ArgumentError, "--stdin-credentials 不能与命令行凭据或密码提示同时使用"
20
+ end
21
+ read_password(values, :ask_password, "NC_DEVICE_PASSWORD", "设备密码", input: input, output: error)
22
+ read_password(values, :ask_netdisco_password, "NETDISCO_PASSWORD", "Netdisco 密码", input: input, output: error)
23
+
24
+ values
25
+ rescue OptionParser::ParseError
26
+ raise ArgumentError, "参数无效:并发数须为 1 至 50,抽样须为 1 至 5;使用 --help 查看参数", cause: nil
27
+ end
28
+
29
+ def self.parser_for(values, output:, program:)
30
+ OptionParser.new do |opts|
31
+ opts.banner = "用法:ruby #{File.basename(program)} [选项](默认全量)"
32
+ add_settings(opts, values)
33
+ add_execution(opts, values)
34
+ opts.on("--ask-password", "隐藏输入设备密码") { values[:ask_password] = true }
35
+ opts.on("--ask-netdisco-password", "隐藏输入 Netdisco 密码") { values[:ask_netdisco_password] = true }
36
+ opts.on("--verbose", "显示逐条登录和命令事件") { values[:verbose] = true }
37
+ opts.on("--json", "在 STDOUT 输出计划和结果 JSON") { values[:json] = true }
38
+ opts.on("--stdin-credentials", "从标准输入读取凭据并等待 RUN") { values[:stdin_credentials] = true }
39
+ opts.on("-h", "--help", "显示帮助") { output.puts opts; values[:help] = true }
40
+ end
41
+ end
42
+
43
+ def self.add_execution(opts, values)
44
+ opts.on("-c", "--concurrency N", Integer, "并发设备数,1 至 50") do |count|
45
+ raise OptionParser::InvalidArgument unless (1..Settings::MAX_CONCURRENCY).cover?(count)
46
+
47
+ values[:environment]["NC_CONCURRENCY"] = count.to_s
48
+ end
49
+ opts.on("--sample N", "--limit-per-vendor N", Integer, "每厂商抽样 1 至 5 台") do |count|
50
+ Planner.validate_limit!(count)
51
+ values[:limit] = count
52
+ end
53
+ opts.on("--tftp-root DIR", "本机 TFTP 服务器文件目录") { |path| values[:tftp_root] = path }
54
+ opts.on("--all", "选择全部就绪设备") { values[:all] = true }
55
+ opts.on("--success-policy POLICY", %w[strict selected verified], "成功判定:strict / selected / verified(TFTP 文件核验)") do |policy|
56
+ values[:success_policy] = policy.to_sym
57
+ end
58
+ end
59
+
60
+ def self.settings(options, env: ENV)
61
+ raise ArgumentError, "--all 不能与抽样同时使用" if options[:all] && options[:limit]
62
+
63
+ environment = environment(options, env: env)
64
+ path = environment["NC_CONFIG"]
65
+ defaults = path && !path.empty? ? ConfigFile.load(path) : {}
66
+ overrides = {}
67
+ { source: "NETDISCO_SOURCE", query: "NETDISCO_QUERY", query_params: "NETDISCO_QUERY_PARAMS",
68
+ max_script_output_bytes: "NC_MAX_SCRIPT_OUTPUT_BYTES", host: "NC_INCLUDE_HOSTS",
69
+ limit: "NC_SAMPLE_PER_VENDOR" }.each do |option, key|
70
+ overrides[key] = options[option].to_s if options.key?(option)
71
+ end
72
+ overrides["NC_SAMPLE_PER_VENDOR"] = nil if options[:all]
73
+ Settings.new(env: environment, defaults: defaults, overrides: overrides)
74
+ end
75
+
76
+ def self.add_settings(opts, values)
77
+ {
78
+ "--username USER" => ["NC_DEVICE_USERNAME", "设备登录用户名"],
79
+ "--password PASSWORD" => ["NC_DEVICE_PASSWORD", "设备登录密码(可改用 --ask-password)"],
80
+ "--enable-password PASSWORD" => ["NC_ENABLE_PASSWORD", "设备提权密码"],
81
+ "--netdisco-url URL" => ["NETDISCO_URL", "Netdisco 地址"],
82
+ "--netdisco-username USER" => ["NETDISCO_USERNAME", "Netdisco 用户名"],
83
+ "--netdisco-password PASSWORD" => ["NETDISCO_PASSWORD", "Netdisco 密码"],
84
+ "--directory DIR" => ["NC_BACKUP_DIRECTORY", "备份目录,相对项目根目录"],
85
+ "--config FILE" => ["NC_CONFIG", "YAML 配置,相对项目根目录"]
86
+ }.each do |flag, (key, description)|
87
+ opts.on(flag, description) do |value|
88
+ raise OptionParser::InvalidArgument if value.empty?
89
+
90
+ values[:environment][key] = value
91
+ end
92
+ end
93
+ opts.on("--host-key-policy POLICY", %w[strict accept_new replace], "主机密钥策略:strict / accept_new / replace(变化时自动替换)") do |policy|
94
+ values[:environment]["NC_HOST_KEY_POLICY"] = policy
95
+ end
96
+ opts.on("--known-hosts FILE", "持久保存主机密钥的文件") do |path|
97
+ values[:environment]["NC_KNOWN_HOSTS"] = path
98
+ end
99
+ end
100
+
101
+ def self.environment(options, env: ENV)
102
+ result = env.to_h.dup
103
+ overrides = options.fetch(:environment)
104
+ # 显式设备参数覆盖对应厂商默认值;未指定的字段继续按原凭据规则解析。
105
+ { "NC_DEVICE_USERNAME" => "USERNAME", "NC_DEVICE_PASSWORD" => "PASSWORD",
106
+ "NC_ENABLE_PASSWORD" => "ENABLE_PASSWORD" }.each do |key, suffix|
107
+ next unless overrides.key?(key)
108
+
109
+ result.delete_if do |name, _|
110
+ name.start_with?("NC_") && name.end_with?("_#{suffix}") &&
111
+ (suffix != "PASSWORD" || !name.end_with?("_ENABLE_PASSWORD"))
112
+ end
113
+ end
114
+ if overrides.key?("NETDISCO_USERNAME") || overrides.key?("NETDISCO_PASSWORD")
115
+ result.delete("NETDISCO_API_KEY")
116
+ end
117
+ result.merge(overrides)
118
+ end
119
+
120
+ def self.read_password(values, flag, key, label, input:, output:)
121
+ return unless values[flag]
122
+
123
+ raise ArgumentError, "密码提示与显式密码不能同时使用" if values[:environment].key?(key)
124
+ raise ArgumentError, "密码提示需要交互终端;自动化任务可使用环境变量或 --stdin-credentials" unless input.tty?
125
+ require "io/console"
126
+ output.print "#{label}:"
127
+ output.flush
128
+ password = input.noecho(&:gets)&.chomp
129
+ output.puts
130
+ raise ArgumentError, "密码不能为空" if password.nil? || password.empty?
131
+
132
+ values[:environment][key] = password
133
+ end
134
+ private_class_method :read_password, :parser_for
135
+ end
136
+ end
137
+ end
138
+ end
139
+ end
@@ -54,7 +54,10 @@ module Net
54
54
  message = Storage::PrivateFile.receipt_error?(exception) ? exception.message : exception.class.name
55
55
  @error.puts "net-connector-backup: #{message}"
56
56
  2
57
- rescue OptionParser::ParseError, ArgumentError, KeyError, Errno::ENOENT,
57
+ rescue OptionParser::ParseError
58
+ @error.puts "net-connector-backup: 参数无效;使用 --help 查看参数"
59
+ 2
60
+ rescue ArgumentError, KeyError, Errno::ENOENT,
58
61
  Psych::Exception, JSON::ParserError, Client::Error => exception
59
62
  @error.puts "net-connector-backup: #{exception.message}"
60
63
  2
@@ -67,21 +70,13 @@ module Net
67
70
 
68
71
  # 文件、环境变量、命令行按优先级覆盖,凭据仍只来自环境变量。
69
72
  def settings_for(options)
70
- defaults = options[:config] ? ConfigFile.load(options[:config]) : {}
71
- values = {}
72
- values["NET_CONNECTOR_BACKUP_DIRECTORY"] = options[:directory] if options[:directory]
73
- values["NET_CONNECTOR_CONCURRENCY"] = options[:concurrency].to_s if options[:concurrency]
74
- values["NET_CONNECTOR_MAX_SCRIPT_OUTPUT_BYTES"] = options[:max_script_output_bytes].to_s if options[:max_script_output_bytes]
75
- values["NET_CONNECTOR_INCLUDE_HOSTS"] = options[:host] if options[:host]
76
- values["NET_CONNECTOR_SAMPLE_PER_VENDOR"] = options[:limit].to_s if options[:limit]
77
- values["NET_CONNECTOR_SAMPLE_PER_VENDOR"] = nil if options[:all]
78
- Settings.new(env: @env, defaults: defaults, overrides: values)
73
+ Options.settings(options, env: @env)
79
74
  end
80
75
 
81
76
  # 导出已有配置不创建清单或设备连接。
82
77
  def export_config(settings, options)
83
78
  path = Storage::SavedConfig.new(directory: settings.backup_directory)
84
- .export(host: options[:export], output: options[:output], io: @output)
79
+ .export(host: options[:export], output: options[:output], io: @output)
85
80
  @output.puts JSON.generate(host: options[:export], output: path) if path
86
81
  end
87
82
 
@@ -107,7 +102,7 @@ module Net
107
102
  fleet.tftp_backup_all(plan: plan, server: settings.tftp_server,
108
103
  concurrency: settings.concurrency,
109
104
  vrfs: settings.tftp_vrfs,
110
- source_files: settings.tftp_source_files, **reporting)
105
+ source_files: settings.tftp_source_files, verification_root: options[:tftp_root] || @env["TFTP_ROOT"], **reporting)
111
106
  else
112
107
  fleet.backup_all(plan: plan, directory: settings.backup_directory,
113
108
  concurrency: settings.concurrency, **reporting)
@@ -116,24 +111,21 @@ module Net
116
111
 
117
112
  # 解析命令行参数并处理帮助及版本信息。
118
113
  def parse_options
119
- options = { config: @env["NET_CONNECTOR_CONFIG"] }
114
+ options = { environment: {} }
120
115
  parser = OptionParser.new do |args|
121
116
  args.banner = "用法:net-connector-backup [--config FILE] [--plan | --show-config | --export IP] [--tftp]"
122
- args.on("--config FILE", "读取不含凭据的 YAML 设置") { |value| options[:config] = value }
117
+ Options.add_settings(args, options)
118
+ Options.add_execution(args, options)
119
+ args.on("--source SOURCE", %w[http postgres], "清单来源:http(默认)或 postgres") { |value| options[:source] = value }
120
+ args.on("--query SQL", "PostgreSQL 清单查询,使用 $1、$2 绑定参数") { |value| options[:query] = value }
121
+ args.on("--query-params JSON", "查询参数的 JSON 数组") { |value| options[:query_params] = value }
123
122
  args.on("--show-config", "显示生效的非敏感设置") { options[:show_config] = true }
124
123
  args.on("--plan", "预览设备清单,不连接设备") { options[:plan] = true }
125
124
  args.on("--tftp", "由设备发起 TFTP 备份") { options[:tftp] = true }
126
125
  args.on("--export IP", "导出已保存的本地配置") { |value| options[:export] = value }
127
126
  args.on("--output FILE", "将导出配置写入私有文件") { |value| options[:output] = value }
128
- args.on("--directory PATH", "备份和报告目录") { |value| options[:directory] = value }
129
127
  args.on("--host IP", "按管理地址选择一台设备") { |value| options[:host] = value }
130
- args.on("--concurrency N", Integer, "最大并发设备数") { |value| options[:concurrency] = value }
131
128
  args.on("--max-script-output-bytes N", Integer, "每个脚本的累计响应字节上限") { |value| options[:max_script_output_bytes] = value }
132
- args.on("--limit-per-vendor N", Integer, "每厂商选择 N 台设备(1 至 5)") do |value|
133
- options[:limit] = value
134
- end
135
- args.on("--all", "选择全部就绪设备") { options[:all] = true }
136
- args.on("--success-policy POLICY", %w[strict selected], "成功判定:strict(默认)或 selected") { |value| options[:success_policy] = value.to_sym }
137
129
  args.on("-v", "--version", "显示版本") do
138
130
  @output.puts Net::Connector::VERSION
139
131
  options[:done] = true
@@ -151,15 +143,19 @@ module Net
151
143
 
152
144
  # 拒绝互斥或缺少依赖的命令行选项。
153
145
  def validate_options!(options)
146
+ raise ArgumentError, "verified policy requires --tftp" if options[:success_policy] == :verified && !options[:tftp]
154
147
  Report.validate_policy!(options.fetch(:success_policy, :strict))
155
148
  Planner.validate_limit!(options[:limit])
156
- Worker.new(concurrency: options[:concurrency]) if options[:concurrency]
149
+ raise ArgumentError, "--tftp-root requires --tftp" if options[:tftp_root] && !options[:tftp]
157
150
  modes = [:show_config, :plan, :export].count { |key| options[key] }
158
151
  raise ArgumentError, "--show-config、--plan 和 --export 只能选择一项" if modes > 1
159
152
  raise ArgumentError, "--output 需要同时指定 --export" if options[:output] && !options[:export]
160
153
  raise ArgumentError, "--all 不能与 --limit-per-vendor 同时使用" if options[:all] && options[:limit]
161
- if options[:export] && (options[:tftp] || options[:all] || options[:limit] || options[:concurrency] || options[:host] ||
162
- options[:success_policy] || options[:max_script_output_bytes])
154
+ if options[:export] && (options[:source] || options[:query] || options[:query_params])
155
+ raise ArgumentError, "--export 不能与清单查询选项同时使用"
156
+ end
157
+ if options[:export] && (options[:tftp] || options[:all] || options[:limit] || options[:environment].key?("NC_CONCURRENCY") || options[:host] ||
158
+ options[:success_policy] || options[:tftp_root] || options[:max_script_output_bytes])
163
159
  raise ArgumentError, "--export 不能与备份选项同时使用"
164
160
  end
165
161
  end
@@ -96,7 +96,7 @@ module Net
96
96
  @options.fetch(:max_pages).times do |index|
97
97
  uri = endpoint("api/v1/search/device")
98
98
  uri.query = URI.encode_www_form(fields: FIELDS.join(","), limit: @options.fetch(:page_size),
99
- offset: index * @options.fetch(:page_size))
99
+ offset: index * @options.fetch(:page_size))
100
100
  page = request_json(uri, authorized_get(uri, token), budget, allow_empty: true, query_fallback: true)
101
101
  validate_rows!(page, budget)
102
102
  return rows.uniq if page.empty?
@@ -258,9 +258,9 @@ module Net
258
258
  row.is_a?(Hash) && present?(row["ip"]) &&
259
259
  FIELDS.drop(1).all? { |field| row[field].nil? || row[field].is_a?(String) }
260
260
  end
261
- unless valid
262
- raise Error.new("Netdisco returned an invalid device inventory", code: :invalid_inventory), cause: nil
263
- end
261
+ return if valid
262
+
263
+ raise Error.new("Netdisco returned an invalid device inventory", code: :invalid_inventory), cause: nil
264
264
  end
265
265
 
266
266
  def validate_credentials!(username, password, api_key)
@@ -268,12 +268,13 @@ module Net
268
268
  (present?(api_key) && username.nil? && password.nil?)
269
269
  raise ArgumentError, "provide either username and password or api_key"
270
270
  end
271
- if [username, password, api_key].compact.any? { |value| value.match?(/[\r\n\x00]/) }
272
- raise ArgumentError, "credentials must be single-line strings"
273
- end
271
+ return unless [username, password, api_key].compact.any? { |value| value.match?(/[\r\n\x00]/) }
272
+
273
+ raise ArgumentError, "credentials must be single-line strings"
274
274
  end
275
275
 
276
276
  def monotonic = Process.clock_gettime(Process::CLOCK_MONOTONIC)
277
+
277
278
  def present?(value) = value.is_a?(String) && !value.strip.empty?
278
279
  end
279
280
  end
@@ -9,7 +9,10 @@ module Net
9
9
  # 命令行使用的无凭据 YAML 配置,环境变量优先。
10
10
  class ConfigFile
11
11
  FIELDS = {
12
- "netdisco" => { "url" => ["NETDISCO_URL", :string],
12
+ "netdisco" => { "source" => ["NETDISCO_SOURCE", :string],
13
+ "query" => ["NETDISCO_QUERY", :string],
14
+ "query_params" => ["NETDISCO_QUERY_PARAMS", :json_array],
15
+ "url" => ["NETDISCO_URL", :string],
13
16
  "page_size" => ["NETDISCO_PAGE_SIZE", :integer],
14
17
  "max_pages" => ["NETDISCO_MAX_PAGES", :integer],
15
18
  "max_response_bytes" => ["NETDISCO_MAX_RESPONSE_BYTES", :integer],
@@ -17,26 +20,27 @@ module Net
17
20
  "max_devices" => ["NETDISCO_MAX_DEVICES", :integer],
18
21
  "inventory_timeout" => ["NETDISCO_INVENTORY_TIMEOUT", :number],
19
22
  "allow_insecure_http" => ["NETDISCO_ALLOW_INSECURE_HTTP", :boolean] },
20
- "backup" => { "directory" => ["NET_CONNECTOR_BACKUP_DIRECTORY", :string],
21
- "concurrency" => ["NET_CONNECTOR_CONCURRENCY", :integer],
22
- "limit_per_vendor" => ["NET_CONNECTOR_SAMPLE_PER_VENDOR", :integer] },
23
- "inventory" => { "include_hosts" => ["NET_CONNECTOR_INCLUDE_HOSTS", :list],
24
- "exclude_hosts" => ["NET_CONNECTOR_EXCLUDE_HOSTS", :list],
25
- "include_vendors" => ["NET_CONNECTOR_INCLUDE_VENDORS", :list],
26
- "vendor_overrides" => ["NET_CONNECTOR_VENDOR_OVERRIDES", :json],
27
- "host_overrides" => ["NET_CONNECTOR_HOST_OVERRIDES", :json],
28
- "device_rules" => ["NET_CONNECTOR_DEVICE_RULES", :json] },
29
- "ssh" => { "protocol" => ["NET_CONNECTOR_PROTOCOL", :string],
30
- "max_script_output_bytes" => ["NET_CONNECTOR_MAX_SCRIPT_OUTPUT_BYTES", :optional_integer],
31
- "known_hosts" => ["NET_CONNECTOR_KNOWN_HOSTS", :string],
32
- "host_key_policy" => ["NET_CONNECTOR_HOST_KEY_POLICY", :string],
33
- "log_directory" => ["NET_CONNECTOR_LOG_DIRECTORY", :string],
34
- "log_level" => ["NET_CONNECTOR_LOG_LEVEL", :string] },
23
+ "backup" => { "directory" => ["NC_BACKUP_DIRECTORY", :string],
24
+ "concurrency" => ["NC_CONCURRENCY", :integer],
25
+ "limit_per_vendor" => ["NC_SAMPLE_PER_VENDOR", :integer] },
26
+ "inventory" => { "include_hosts" => ["NC_INCLUDE_HOSTS", :list],
27
+ "exclude_hosts" => ["NC_EXCLUDE_HOSTS", :list],
28
+ "include_vendors" => ["NC_INCLUDE_VENDORS", :list],
29
+ "vendor_overrides" => ["NC_VENDOR_OVERRIDES", :json],
30
+ "host_overrides" => ["NC_HOST_OVERRIDES", :json],
31
+ "device_rules" => ["NC_DEVICE_RULES", :json] },
32
+ "ssh" => { "protocol" => ["NC_PROTOCOL", :string],
33
+ "vendor_protocols" => ["NC_VENDOR_PROTOCOLS", :json],
34
+ "max_script_output_bytes" => ["NC_MAX_SCRIPT_OUTPUT_BYTES", :optional_integer],
35
+ "known_hosts" => ["NC_KNOWN_HOSTS", :string],
36
+ "host_key_policy" => ["NC_HOST_KEY_POLICY", :string],
37
+ "log_directory" => ["NC_LOG_DIRECTORY", :string],
38
+ "log_level" => ["NC_LOG_LEVEL", :string] },
35
39
  "tftp" => { "server" => ["TFTP_HOST", :string],
36
- "vrfs" => ["NET_CONNECTOR_TFTP_VRFS", :json],
37
- "h3c_source_file" => ["NET_CONNECTOR_H3C_TFTP_SOURCE_FILE", :string],
38
- "h3c_wireless_source_file" => ["NET_CONNECTOR_H3C_WIRELESS_TFTP_SOURCE_FILE", :string],
39
- "huawei_source_file" => ["NET_CONNECTOR_HUAWEI_TFTP_SOURCE_FILE", :string] }
40
+ "vrfs" => ["NC_TFTP_VRFS", :json],
41
+ "h3c_source_file" => ["NC_H3C_TFTP_SOURCE_FILE", :string],
42
+ "h3c_wireless_source_file" => ["NC_H3C_WIRELESS_TFTP_SOURCE_FILE", :string],
43
+ "huawei_source_file" => ["NC_HUAWEI_TFTP_SOURCE_FILE", :string] }
40
44
  }.freeze
41
45
 
42
46
  # 安全读取 YAML 配置并映射到环境变量格式。
@@ -94,8 +98,8 @@ module Net
94
98
  end
95
99
 
96
100
  value.join(",")
97
- when :json
98
- expected = name == "device_rules" ? Array : Hash
101
+ when :json, :json_array
102
+ expected = type == :json_array || name == "device_rules" ? Array : Hash
99
103
  raise ArgumentError, "#{section}.#{name} must be a #{expected}" unless value.is_a?(expected)
100
104
 
101
105
  JSON.generate(value)
@@ -0,0 +1,38 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "json"
4
+ require_relative "client"
5
+
6
+ # 清单连接与设备凭据来源;业务入口负责规划和执行备份。
7
+ module Net
8
+ module Connector
9
+ module Netdisco
10
+ # 将调用方提供的输入解析为清单客户端和设备凭据 resolver。
11
+ class Connection
12
+ def self.build(settings, stdin_credentials: false, input: $stdin)
13
+ unless stdin_credentials
14
+ return [settings.client, settings.method(:device_credentials_for)]
15
+ end
16
+
17
+ begin
18
+ values = JSON.parse(input.gets || "")
19
+ rescue JSON::ParserError
20
+ raise ArgumentError, "凭据 JSON 无效", cause: nil
21
+ end
22
+ keys = %w[netdisco_username netdisco_password device_username device_password]
23
+ raise ArgumentError, "需要四个非空凭据字符串" unless values.is_a?(Hash) && keys.all? { |key| values[key].is_a?(String) && !values[key].strip.empty? }
24
+
25
+ client = settings.client(credentials: {
26
+ username: values.fetch("netdisco_username"), password: values.fetch("netdisco_password")
27
+ })
28
+ credentials = lambda do |device|
29
+ inherited = device ? settings.device_credentials_for(device) : nil
30
+ (inherited || {}).merge(username: values.fetch("device_username"),
31
+ password: values.fetch("device_password"))
32
+ end
33
+ [client, credentials]
34
+ end
35
+ end
36
+ end
37
+ end
38
+ end
@@ -0,0 +1,197 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "timeout"
4
+ require_relative "client"
5
+
6
+ module Net
7
+ module Connector
8
+ module Netdisco
9
+ # SQL 负责业务选择和列别名;适配器只管理只读查询、清单预算与连接生命周期。
10
+ class DatabaseClient
11
+ CONNECTION_ENV = {
12
+ host: "NETDISCO_DB_HOST", port: "NETDISCO_DB_PORT", dbname: "NETDISCO_DB_NAME",
13
+ user: "NETDISCO_DB_USER", password: "NETDISCO_DB_PASS",
14
+ sslmode: "NETDISCO_DB_SSLMODE", sslrootcert: "NETDISCO_DB_SSLROOTCERT",
15
+ connect_timeout: "NETDISCO_DB_CONNECT_TIMEOUT"
16
+ }.freeze
17
+ REQUIRED_CONNECTION_KEYS = %i[host dbname user password].freeze
18
+ DEFAULTS = Client::DEFAULTS.except(:allow_insecure_http).freeze
19
+
20
+ def self.options(query:, query_params: [], **values)
21
+ unless query.is_a?(String) && !query.strip.empty? && !query.include?("\0")
22
+ raise ArgumentError, "query must be a nonempty SQL string without NUL"
23
+ end
24
+ unless query_params.is_a?(Array) && query_params.all? { |value|
25
+ value.nil? || value == true || value == false || value.is_a?(Integer) ||
26
+ (value.is_a?(Float) && value.finite?) || (value.is_a?(String) && !value.include?("\0"))
27
+ }
28
+ raise ArgumentError, "query_params must be an array of JSON scalar values without NUL"
29
+ end
30
+ raise ArgumentError, "unknown database inventory options" unless (values.keys - DEFAULTS.keys).empty?
31
+
32
+ Client.options(**values).slice(*DEFAULTS.keys).merge(
33
+ query: query.dup.freeze,
34
+ query_params: query_params.map { |value| value.is_a?(String) ? value.dup.freeze : value }.freeze
35
+ ).freeze
36
+ end
37
+
38
+ def initialize(connection_options:, **options)
39
+ @options = self.class.options(**options)
40
+ @connection_options = validate_connection_options!(connection_options)
41
+ end
42
+
43
+ # 每次查询独占连接和游标;只有全部结果验证成功后才把清单交给 Fleet。
44
+ def devices
45
+ require "pg"
46
+ budget = InventoryBudget.new(@options, clock: method(:monotonic))
47
+ connection = nil
48
+ connected = false
49
+ begin
50
+ Timeout.timeout(budget.remaining, Client::InventoryTimeout) do
51
+ # 从握手开始持有连接;PG.connect 尚未返回时被超时中断,调用方无法负责关闭它。
52
+ connection = PG::Connection.connect_start(@connection_options)
53
+ # PostgreSQL NOTICE 可能包含 SQL 或数据,不交给 libpq 默认的 stderr 输出器。
54
+ connection.set_notice_processor { |_notice| nil }
55
+ wait_for_connection(connection, budget)
56
+ connected = true
57
+ # 使用 pg 可中断的 Ruby 等待接口;底层 socket 仍以非阻塞方式工作。
58
+ connection.setnonblocking(false)
59
+ connection.set_client_encoding("UTF8")
60
+ connection.exec("BEGIN READ ONLY").clear
61
+ set_statement_timeout(connection, budget)
62
+ # 即使没有参数也走扩展查询协议,由 PostgreSQL 拒绝多语句和非查询命令。
63
+ connection.exec_params("DECLARE net_connector_inventory NO SCROLL CURSOR FOR\n#{@options.fetch(:query)}",
64
+ @options.fetch(:query_params)).clear
65
+ rows = inventory(connection, budget)
66
+ connection.exec("ROLLBACK").clear
67
+ budget.remaining
68
+ rows
69
+ end
70
+ rescue Client::Error => error
71
+ raise error, cause: nil
72
+ rescue PG::QueryCanceled
73
+ raise Client::InventoryTimeout, cause: nil
74
+ rescue PG::Error, IOError, SystemCallError
75
+ code = connected ? :query_failed : :connection_failed
76
+ raise Client::Error.new("Netdisco database #{code.to_s.tr("_", " ")}", code: code), cause: nil
77
+ ensure
78
+ connection&.finish
79
+ end
80
+ end
81
+
82
+ def inspect = "#<#{self.class}>"
83
+
84
+ private
85
+
86
+ def wait_for_connection(connection, budget)
87
+ deadline = monotonic + @connection_options.fetch(:connect_timeout, budget.remaining).to_f
88
+ loop do
89
+ status = connection.connect_poll
90
+ return if status == PG::PGRES_POLLING_OK
91
+ raise PG::ConnectionBad unless [PG::PGRES_POLLING_READING, PG::PGRES_POLLING_WRITING].include?(status)
92
+
93
+ remaining = [budget.remaining, deadline - monotonic].min
94
+ raise PG::ConnectionBad if remaining <= 0
95
+
96
+ socket = connection.socket_io
97
+ readers = status == PG::PGRES_POLLING_READING ? [socket] : nil
98
+ writers = status == PG::PGRES_POLLING_WRITING ? [socket] : nil
99
+ next if IO.select(readers, writers, [socket], remaining)
100
+
101
+ budget.remaining
102
+ raise PG::ConnectionBad
103
+ end
104
+ end
105
+
106
+ def validate_connection_options!(options)
107
+ unless options.is_a?(Hash) && (options.keys - CONNECTION_ENV.keys).empty?
108
+ raise ArgumentError, "connection_options must contain supported PostgreSQL connection keys"
109
+ end
110
+ REQUIRED_CONNECTION_KEYS.each do |key|
111
+ value = options[key]
112
+ unless value.is_a?(String) && !value.strip.empty?
113
+ raise ArgumentError, "#{CONNECTION_ENV.fetch(key)} is required"
114
+ end
115
+ end
116
+ options.each do |key, value|
117
+ valid = value.is_a?(String) && !value.empty? && !value.include?("\0")
118
+ valid ||= %i[port connect_timeout].include?(key) && value.is_a?(Integer)
119
+ raise ArgumentError, "#{CONNECTION_ENV.fetch(key)} is invalid" unless valid
120
+ end
121
+ %i[port connect_timeout].each do |key|
122
+ next unless options.key?(key)
123
+
124
+ value = options.fetch(key).to_s
125
+ maximum = key == :port ? 65_535 : 2_147_483_647
126
+ unless value.match?(/\A\d+\z/) && value.to_i.between?(1, maximum)
127
+ raise ArgumentError, "#{CONNECTION_ENV.fetch(key)} must be a positive integer within range"
128
+ end
129
+ end
130
+ if options[:sslmode] && !%w[disable allow prefer require verify-ca verify-full].include?(options[:sslmode])
131
+ raise ArgumentError, "NETDISCO_DB_SSLMODE is invalid"
132
+ end
133
+ # libpq 会把 dbname 中的 URI/conninfo 再解释为连接参数,不能借此绕过凭据来源。
134
+ if options.fetch(:dbname).match?(/=|\Apostgres(?:ql)?:\/\//)
135
+ raise ArgumentError, "NETDISCO_DB_NAME must be a database name, not a connection string"
136
+ end
137
+ options.transform_values { |value| value.is_a?(String) ? value.dup.freeze : value }.freeze
138
+ end
139
+
140
+ def set_statement_timeout(connection, budget)
141
+ milliseconds = [(budget.remaining * 1000).ceil, 2_147_483_647].min
142
+ connection.exec_params("SELECT set_config('statement_timeout', $1, true)", [milliseconds.to_s]).clear
143
+ end
144
+
145
+ def inventory(connection, budget)
146
+ rows = []
147
+ @options.fetch(:max_pages).times do
148
+ set_statement_timeout(connection, budget)
149
+ count = fetch_page(connection, rows, budget)
150
+ return rows if count < @options.fetch(:page_size)
151
+ end
152
+ raise Client::Error.new("Netdisco inventory exceeded max_pages", code: :max_pages), cause: nil
153
+ end
154
+
155
+ # 单行模式避免 libpq 先缓存整页;Ruby 只积累已经通过预算检查的白名单字段。
156
+ def fetch_page(connection, rows, budget)
157
+ connection.send_query_params("FETCH FORWARD #{@options.fetch(:page_size)} FROM net_connector_inventory", [])
158
+ connection.set_single_row_mode
159
+ count = 0
160
+ response_bytes = 0
161
+ while (result = connection.get_result)
162
+ begin
163
+ result.check
164
+ response_bytes = append_rows(result, rows, budget, response_bytes)
165
+ count += result.ntuples
166
+ ensure
167
+ result.clear
168
+ end
169
+ end
170
+ count
171
+ end
172
+
173
+ def append_rows(result, rows, budget, response_bytes)
174
+ fields = result.fields
175
+ unless fields.include?("ip") && fields.uniq == fields
176
+ raise Client::Error.new("Netdisco database query must return unique column names including ip",
177
+ code: :invalid_inventory), cause: nil
178
+ end
179
+ budget.consume_devices(result.ntuples)
180
+ result.each do |row|
181
+ # 未选用的列也占传输预算,但不会进入清单、计划或报告。
182
+ size = row.sum { |key, value| key.bytesize + (value&.bytesize || 0) }
183
+ budget.consume_bytes(size, response_bytes: response_bytes)
184
+ response_bytes += size
185
+ unless row["ip"].is_a?(String) && !row["ip"].strip.empty?
186
+ raise Client::Error.new("Netdisco returned an invalid device inventory", code: :invalid_inventory), cause: nil
187
+ end
188
+ rows << row.slice(*Client::FIELDS)
189
+ end
190
+ response_bytes
191
+ end
192
+
193
+ def monotonic = Process.clock_gettime(Process::CLOCK_MONOTONIC)
194
+ end
195
+ end
196
+ end
197
+ end
@@ -44,7 +44,17 @@ module Net
44
44
  def ready? = issue.nil?
45
45
 
46
46
  # 生成不随名称或厂商信息改变的本地文本文件名。
47
- def backup_filename = Storage::SavedConfig.filename(host)
47
+ def backup_filename(style: :ip)
48
+ address = Storage::SavedConfig.filename(host)
49
+ return address if style == :ip
50
+ raise ArgumentError, "filename style must be ip or hostname_ip" unless style == :hostname_ip
51
+
52
+ label = name.to_s.encode("UTF-8", invalid: :replace, undef: :replace, replace: "-")
53
+ .gsub(/[^\p{L}\p{N}._-]+/, "-").byteslice(0, 160).scrub("")
54
+ .gsub(/\A[._-]+|[._-]+\z/, "")
55
+ label = "unnamed" if label.empty?
56
+ "#{label}-#{address}"
57
+ end
48
58
 
49
59
  # 生成设备 CLI 可用的 ASCII 远端文件名。
50
60
  def tftp_filename