net-connector 0.5.0 → 0.7.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (60) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +58 -0
  3. data/CONTRIBUTING.md +20 -0
  4. data/README.md +153 -42
  5. data/docs/VERIFICATION.md +42 -1
  6. data/docs/architecture.md +20 -1
  7. data/examples/backup.yml +40 -0
  8. data/examples/inventory_sql.yml +16 -0
  9. data/lib/net/connector/device/base.rb +30 -38
  10. data/lib/net/connector/device/profile/builder.rb +8 -8
  11. data/lib/net/connector/device/profile.rb +2 -2
  12. data/lib/net/connector/device/running_config.rb +2 -2
  13. data/lib/net/connector/device/tftp/receipt.rb +12 -3
  14. data/lib/net/connector/device/tftp.rb +7 -7
  15. data/lib/net/connector/device/topology.rb +4 -4
  16. data/lib/net/connector/engine/authentication.rb +1 -1
  17. data/lib/net/connector/engine/command.rb +2 -2
  18. data/lib/net/connector/engine/configuration.rb +5 -2
  19. data/lib/net/connector/engine/dialogue.rb +4 -4
  20. data/lib/net/connector/engine/execution.rb +27 -16
  21. data/lib/net/connector/engine/known_hosts.rb +112 -0
  22. data/lib/net/connector/engine/log.rb +10 -7
  23. data/lib/net/connector/engine/session.rb +117 -128
  24. data/lib/net/connector/engine/transport.rb +32 -7
  25. data/lib/net/connector/netdisco/backup_run.rb +132 -0
  26. data/lib/net/connector/netdisco/batch.rb +1 -1
  27. data/lib/net/connector/netdisco/cli/options.rb +139 -0
  28. data/lib/net/connector/netdisco/cli.rb +20 -24
  29. data/lib/net/connector/netdisco/client.rb +8 -7
  30. data/lib/net/connector/netdisco/config_file.rb +26 -22
  31. data/lib/net/connector/netdisco/connection.rb +38 -0
  32. data/lib/net/connector/netdisco/database_client.rb +197 -0
  33. data/lib/net/connector/netdisco/device.rb +11 -1
  34. data/lib/net/connector/netdisco/fleet.rb +68 -47
  35. data/lib/net/connector/netdisco/inventory_budget.rb +4 -4
  36. data/lib/net/connector/netdisco/plan.rb +14 -9
  37. data/lib/net/connector/netdisco/planner.rb +18 -14
  38. data/lib/net/connector/netdisco/progress.rb +253 -0
  39. data/lib/net/connector/netdisco/report/files.rb +41 -0
  40. data/lib/net/connector/netdisco/report/text.rb +45 -0
  41. data/lib/net/connector/netdisco/report.rb +53 -31
  42. data/lib/net/connector/netdisco/result_store.rb +3 -1
  43. data/lib/net/connector/netdisco/rules.rb +10 -2
  44. data/lib/net/connector/netdisco/settings.rb +136 -63
  45. data/lib/net/connector/netdisco/tftp_archive.rb +175 -0
  46. data/lib/net/connector/netdisco/tftp_history.rb +16 -0
  47. data/lib/net/connector/netdisco/tftp_verification.rb +58 -0
  48. data/lib/net/connector/netdisco/worker.rb +8 -3
  49. data/lib/net/connector/netdisco.rb +11 -0
  50. data/lib/net/connector/storage/backup_lock.rb +1 -1
  51. data/lib/net/connector/storage/batch_directory.rb +29 -0
  52. data/lib/net/connector/storage.rb +1 -0
  53. data/lib/net/connector/vendor/h3c.rb +2 -1
  54. data/lib/net/connector/vendor/hillstone/tftp_backup.rb +3 -3
  55. data/lib/net/connector/vendor/palo_alto/tftp_backup.rb +3 -3
  56. data/lib/net/connector/vendor/radware/running_config.rb +19 -0
  57. data/lib/net/connector/vendor/radware.rb +4 -3
  58. data/lib/net/connector/version.rb +1 -1
  59. data/lib/net/connector.rb +2 -2
  60. metadata +57 -104
@@ -13,7 +13,7 @@ module Net
13
13
 
14
14
  # 组装清单客户端、规则、凭据与结果存储。
15
15
  def initialize(client: nil, settings: Settings.new, rules: nil, credentials: nil, connector_factory: nil,
16
- result_store: ResultStore::Text.new)
16
+ result_store: ResultStore::Json.new)
17
17
  @settings = settings
18
18
  @client = client
19
19
  @rules = rules
@@ -35,46 +35,53 @@ module Net
35
35
  end
36
36
 
37
37
  # 规划 TFTP 配置备份的设备集合。
38
- def plan_tftp_backup(limit_per_vendor: 5)
39
- build_plan(:tftp, limit_per_vendor, @settings.snapshot(mode: :tftp))
38
+ def plan_tftp_backup(limit_per_vendor: 5, allow_fixed_name_reuse: false)
39
+ build_plan(:tftp, limit_per_vendor, @settings.snapshot(mode: :tftp),
40
+ allow_fixed_name_reuse: allow_fixed_name_reuse)
40
41
  end
41
42
 
42
43
  # 先拉取完整清单,再并发采集并保存设备配置。
43
44
  def backup_all(directory: DEFAULT_SETTING, concurrency: DEFAULT_SETTING,
44
45
  limit_per_vendor: nil, plan: nil, on_start: nil, on_result: nil, on_change: nil,
45
- success_policy: :strict)
46
- reporting = Report.validate_policy!(success_policy)
47
- policy = @settings.snapshot(mode: :backup)
48
- directory = policy.backup_directory if directory.equal?(DEFAULT_SETTING)
49
- concurrency = policy.concurrency if concurrency.equal?(DEFAULT_SETTING)
46
+ success_policy: :strict, filename_style: :ip)
47
+ raise ArgumentError, "filename style must be ip or hostname_ip" unless %i[ip hostname_ip].include?(filename_style)
48
+
49
+ Report.validate_policy!(success_policy)
50
+ raise ArgumentError, "verified policy requires TFTP mode" if success_policy == :verified
51
+ settings_snapshot = @settings.snapshot(mode: :backup)
52
+ directory = settings_snapshot.backup_directory if directory.equal?(DEFAULT_SETTING)
53
+ concurrency = settings_snapshot.concurrency if concurrency.equal?(DEFAULT_SETTING)
50
54
  raise ArgumentError, "directory must be a nonempty String" unless directory.is_a?(String) && !directory.empty?
51
- Worker.new(concurrency: concurrency)
55
+ Worker.validate_concurrency!(concurrency)
52
56
  Planner.validate_limit!(limit_per_vendor)
53
57
  validate_callback!(on_result, :on_result)
54
58
  validate_callback!(on_start, :on_start)
55
59
  validate_callback!(on_change, :on_change)
56
60
 
57
- plan ||= build_plan(:backup, limit_per_vendor, policy)
61
+ plan ||= build_plan(:backup, limit_per_vendor, settings_snapshot)
58
62
  validate_plan!(plan, :backup)
59
63
  callbacks = [on_result]
60
- callbacks << ->(item) { on_change.call(item) if item.backup.is_a?(Backup) && item.backup.changed? } if on_change
64
+ callbacks << ->(outcome) { on_change.call(outcome) if outcome.backup.is_a?(Backup) && outcome.backup.changed? } if on_change
61
65
  target_directory = File.expand_path(directory)
62
66
  run_batch(:backup, plan, concurrency: concurrency, report_directory: directory,
63
- output_directory: target_directory, policy: policy, reporting: reporting,
67
+ output_directory: target_directory, settings_snapshot: settings_snapshot, success_policy: success_policy,
64
68
  on_start: on_start, on_result: callbacks.compact) do |device, log_directory|
65
- backup_one(device, target_directory, log_directory, policy)
69
+ backup_device(device, target_directory, log_directory, settings_snapshot, filename_style: filename_style)
66
70
  end
67
71
  end
68
72
 
69
73
  # 让设备主动导出配置,在执行前检查同批目标文件名冲突。
70
74
  def tftp_backup_all(server:, source_files: {}, concurrency: DEFAULT_SETTING,
71
75
  limit_per_vendor: 5, vrfs: {}, on_start: nil, on_result: nil, plan: nil,
72
- report_directory: DEFAULT_SETTING, success_policy: :strict)
73
- reporting = Report.validate_policy!(success_policy)
74
- policy = @settings.snapshot(mode: :tftp)
75
- concurrency = policy.concurrency if concurrency.equal?(DEFAULT_SETTING)
76
- report_directory = policy.backup_directory if report_directory.equal?(DEFAULT_SETTING)
77
- Worker.new(concurrency: concurrency)
76
+ report_directory: DEFAULT_SETTING, success_policy: :strict, verification_root: nil,
77
+ preserve_history: false)
78
+ Report.validate_policy!(success_policy)
79
+ raise ArgumentError, "verified policy requires a local TFTP directory" if success_policy == :verified && !verification_root
80
+ verification = TftpVerification.new(root: verification_root)
81
+ settings_snapshot = @settings.snapshot(mode: :tftp)
82
+ concurrency = settings_snapshot.concurrency if concurrency.equal?(DEFAULT_SETTING)
83
+ report_directory = settings_snapshot.backup_directory if report_directory.equal?(DEFAULT_SETTING)
84
+ Worker.validate_concurrency!(concurrency)
78
85
  server = TftpTarget.new(host: server, path: "preflight.cfg").host
79
86
  raise ArgumentError, "source_files must be a Hash" unless source_files.is_a?(Hash)
80
87
  source_files.each_value { |source_file| TftpTarget.validate_source_file!(source_file) }
@@ -84,40 +91,53 @@ module Net
84
91
  validate_callback!(on_result, :on_result)
85
92
  validate_callback!(on_start, :on_start)
86
93
 
87
- plan ||= build_plan(:tftp, limit_per_vendor, policy)
94
+ plan ||= build_plan(:tftp, limit_per_vendor, settings_snapshot,
95
+ allow_fixed_name_reuse: preserve_history && !verification_root.nil?)
88
96
  validate_plan!(plan, :tftp)
97
+ if plan.allow_fixed_name_reuse && (!preserve_history || !verification_root)
98
+ raise ArgumentError, "fixed-name TFTP uploads require local history archival"
99
+ end
100
+ archive = TftpArchive.new(directory: report_directory, root: verification_root) if preserve_history
89
101
  run_batch(:tftp, plan, concurrency: concurrency, report_directory: report_directory,
90
- policy: policy, reporting: reporting, on_start: on_start, on_result: on_result) do |device, log_directory|
91
- tftp_backup_one(device, server, source_files, log_directory, vrfs, policy)
102
+ settings_snapshot: settings_snapshot, success_policy: success_policy, on_start: on_start, on_result: on_result,
103
+ verification: preserve_history ? nil : verification) do |device, log_directory|
104
+ if archive
105
+ archive.upload_and_archive(device, started_at: Time.now.utc) do |remote_filename|
106
+ tftp_backup_device(device, server, source_files, log_directory, vrfs, settings_snapshot, path: remote_filename)
107
+ end
108
+ else
109
+ tftp_backup_device(device, server, source_files, log_directory, vrfs, settings_snapshot)
110
+ end
92
111
  end
93
112
  end
94
113
 
95
114
  private
96
115
 
97
116
  # 策略在清单请求之前固定;注入的清单客户端仍由调用方负责其自身生命周期。
98
- def inventory(policy)
99
- rows = (@client || @settings.client(policy: policy)).devices
117
+ def inventory(settings_snapshot)
118
+ rows = (@client || @settings.client(policy: settings_snapshot)).devices
100
119
  raise Client::Error, "Netdisco returned an invalid device inventory" unless rows.is_a?(Array)
101
120
 
102
- rules = @rules || policy.rules
121
+ rules = @rules || settings_snapshot.rules
103
122
  devices = rows.map { |row| Device.from_row(row, rules: rules) }
104
123
  duplicates = devices.group_by(&:host).reject { |host, entries| host.nil? || entries.size == 1 }
105
124
  devices.map { |device| duplicates.key?(device.host) ? device.with(issue: :duplicate_host) : device }.freeze
106
125
  end
107
126
 
108
- def build_plan(mode, limit, policy)
127
+ def build_plan(mode, limit, settings_snapshot, allow_fixed_name_reuse: false)
109
128
  Planner.validate_limit!(limit)
110
- Planner.new(inventory(policy)).call(mode: mode, limit_per_vendor: limit)
129
+ Planner.new(inventory(settings_snapshot)).call(mode: mode, limit_per_vendor: limit,
130
+ allow_fixed_name_reuse: allow_fixed_name_reuse)
111
131
  end
112
132
 
113
133
  # 统一组织设备任务、日志目录与报告;Worker 保证单台设备失败不影响其他设备。
114
- def run_batch(mode, plan, concurrency:, report_directory:, policy:, reporting:, on_start:, on_result:, output_directory: nil)
134
+ def run_batch(mode, plan, concurrency:, report_directory:, settings_snapshot:, success_policy:, on_start:, on_result:, output_directory: nil, verification: nil)
115
135
  worker = Worker.new(concurrency: concurrency)
116
136
  started_at = Time.now.utc
117
137
  started = monotonic
118
138
  outcomes = plan.outcomes.dup
119
139
  FileUtils.mkdir_p(output_directory, mode: 0o700) if output_directory && !plan.ready.empty?
120
- log_directory = policy.log_directory
140
+ log_directory = settings_snapshot.log_directory
121
141
  FileUtils.mkdir_p(log_directory, mode: 0o700) if log_directory && !plan.ready.empty?
122
142
  callback_errors = worker.run(plan.ready, outcomes: outcomes, on_start: on_start, on_result: on_result,
123
143
  on_error: ->(device, error) { build_outcome(device, :failed, error: error) }) do |device|
@@ -126,8 +146,9 @@ module Net
126
146
  batch = Batch.new(mode: mode, outcomes: outcomes.freeze, started_at: started_at,
127
147
  finished_at: Time.now.utc, callback_errors: callback_errors,
128
148
  report_location: nil, report_error: nil)
129
- batch = Report.new(batch, policy: reporting, duration_ms: ((monotonic - started) * 1000).round)
130
- save_report(batch, directory: report_directory)
149
+ report = Report.new(batch, policy: success_policy, duration_ms: ((monotonic - started) * 1000).round, log_directory: log_directory)
150
+ report = verification.call(report) if verification
151
+ save_report(report, directory: report_directory)
131
152
  end
132
153
 
133
154
  def monotonic = Process.clock_gettime(Process::CLOCK_MONOTONIC)
@@ -138,16 +159,16 @@ module Net
138
159
  end
139
160
 
140
161
  # 保存批量报告,记录写入失败而不覆盖设备结果。
141
- def save_report(batch, directory:)
142
- return batch unless @result_store
162
+ def save_report(report, directory:)
163
+ return report unless @result_store
143
164
 
144
- batch.with(report_location: @result_store.write(batch, directory: directory))
165
+ report.with(report_location: @result_store.write(report, directory: directory))
145
166
  rescue Storage::PrivateFile::WriteError => error
146
167
  # 报告也可能已经原子替换;保留已知位置,不将持久性失败误写成完全没有产物。
147
168
  location = error.receipt.path if Storage::PrivateFile.receipt_error?(error) && error.receipt.committed?
148
- batch.with_report_error(error, location: location)
169
+ report.with_report_error(error, location: location)
149
170
  rescue StandardError => error
150
- batch.with_report_error(error)
171
+ report.with_report_error(error)
151
172
  end
152
173
 
153
174
  # 确认传入计划属于当前备份模式。
@@ -165,30 +186,30 @@ module Net
165
186
  end
166
187
 
167
188
  # 执行单台设备的 TFTP 导出。
168
- def tftp_backup_one(device, server, source_files, log_directory, vrfs, policy)
169
- run_one(device, log_directory, policy, success_status: :reported_uploaded,
170
- close_error_status: :reported_with_error, backup_class: TftpReceipt) do |connector|
189
+ def tftp_backup_device(device, server, source_files, log_directory, vrfs, settings_snapshot, path: device.tftp_filename)
190
+ run_device(device, log_directory, settings_snapshot, success_status: :reported_uploaded,
191
+ partial_status: :reported_with_error, backup_class: TftpReceipt) do |connector|
171
192
  source_file = source_files.fetch(device.vendor, nil)
172
- transfer_settings = { host: server, path: device.tftp_filename, source_file: source_file }
193
+ transfer_settings = { host: server, path: path, source_file: source_file }
173
194
  transfer_settings[:vrf] = vrfs.fetch(device.vendor) if vrfs.key?(device.vendor)
174
195
  connector.tftp_backup(**transfer_settings)
175
196
  end
176
197
  end
177
198
 
178
199
  # 执行单台设备的本地配置采集。
179
- def backup_one(device, directory, log_directory, policy)
180
- destination = File.join(directory, device.backup_filename)
200
+ def backup_device(device, directory, log_directory, settings_snapshot, filename_style: :ip)
201
+ destination = File.join(directory, device.backup_filename(style: filename_style))
181
202
  Storage::BackupLock.synchronize(destination, host: device.host) do |path_lock|
182
203
  Storage::SafeFile.open(destination, missing: true) { |_file, _stat| nil }
183
- run_one(device, log_directory, policy, success_status: :backed_up,
184
- close_error_status: :saved_with_error, backup_class: Backup) do |connector|
204
+ run_device(device, log_directory, settings_snapshot, success_status: :backed_up,
205
+ partial_status: :saved_with_error, backup_class: Backup) do |connector|
185
206
  path_lock.with_delegated_lock { connector.backup(path: destination) }
186
207
  end
187
208
  end
188
209
  end
189
210
 
190
211
  # 隔离单台设备的凭据、连接、操作和关闭异常。
191
- def run_one(device, log_directory, policy, success_status:, close_error_status:, backup_class:)
212
+ def run_device(device, log_directory, settings_snapshot, success_status:, partial_status:, backup_class:)
192
213
  connection_settings = @credentials ? @credentials.call(device) : @settings.device_credentials_for(device)
193
214
  return build_outcome(device, :missing_credentials) if connection_settings.nil?
194
215
  unless connection_settings.is_a?(Hash) && connection_settings.keys.all?(Symbol) &&
@@ -197,7 +218,7 @@ module Net
197
218
  raise ArgumentError, "credential resolver must return settings with username and without host"
198
219
  end
199
220
 
200
- connection_settings = policy.connection_options(device.vendor).merge(connection_settings)
221
+ connection_settings = settings_snapshot.connection_options(device.vendor).merge(connection_settings)
201
222
  log_name = device.host.tr(":", "_")
202
223
  if log_directory
203
224
  connection_settings = connection_settings.merge(log_file: File.join(log_directory, "#{log_name}.log"))
@@ -206,7 +227,7 @@ module Net
206
227
  backup, failure = perform_one(connector, backup_class) { yield connector }
207
228
  return build_outcome(device, success_status, backup: backup) unless failure
208
229
 
209
- build_outcome(device, backup ? close_error_status : :failed, backup: backup, error: failure)
230
+ build_outcome(device, backup ? partial_status : :failed, backup: backup, error: failure)
210
231
  rescue StandardError => error
211
232
  build_outcome(device, :failed, error: error)
212
233
  end
@@ -23,21 +23,21 @@ module Net
23
23
  # 在追加正文前计数;不能靠 Content-Length 或 JSON 解析后的大小限制内存。
24
24
  def consume_bytes(size, response_bytes:)
25
25
  remaining
26
- check_limit!(:max_response_bytes, response_bytes + size)
27
- check_limit!(:max_inventory_bytes, @bytes + size)
26
+ validate_limit!(:max_response_bytes, response_bytes + size)
27
+ validate_limit!(:max_inventory_bytes, @bytes + size)
28
28
  @bytes += size
29
29
  end
30
30
 
31
31
  # 去重前的记录同样占用清单内存;兼容查询也使用这个累计计数。
32
32
  def consume_devices(size)
33
33
  remaining
34
- check_limit!(:max_devices, @devices + size)
34
+ validate_limit!(:max_devices, @devices + size)
35
35
  @devices += size
36
36
  end
37
37
 
38
38
  private
39
39
 
40
- def check_limit!(name, value)
40
+ def validate_limit!(name, value)
41
41
  return if value <= @options.fetch(name)
42
42
 
43
43
  raise Client::Error.new("Netdisco inventory exceeded #{name}", code: name), cause: nil
@@ -4,15 +4,18 @@ module Net
4
4
  module Connector
5
5
  module Netdisco
6
6
  # 固定一份清单上的选择结果与跳过原因,供预览和执行复用。
7
- Plan = Data.define(:mode, :inventory, :ready, :outcomes) do
7
+ Plan = Data.define(:mode, :inventory, :ready, :outcomes, :allow_fixed_name_reuse) do
8
8
  # Data 只冻结对象自身;这里复制并冻结可能由调用方修改的计划容器。
9
- def initialize(mode:, inventory:, ready:, outcomes:)
9
+ def initialize(mode:, inventory:, ready:, outcomes:, allow_fixed_name_reuse: false)
10
10
  unless inventory.is_a?(Array) && ready.is_a?(Array) && outcomes.is_a?(Array)
11
11
  raise ArgumentError, "plan inventory, ready tasks, and outcomes must be Arrays"
12
12
  end
13
13
 
14
14
  tasks = ready.map { |task| task.is_a?(Array) ? task.dup.freeze : task }.freeze
15
- super(mode: mode, inventory: inventory.dup.freeze, ready: tasks, outcomes: outcomes.dup.freeze)
15
+ raise ArgumentError, "allow_fixed_name_reuse must be boolean" unless [true, false].include?(allow_fixed_name_reuse)
16
+
17
+ super(mode: mode, inventory: inventory.dup.freeze, ready: tasks, outcomes: outcomes.dup.freeze,
18
+ allow_fixed_name_reuse: allow_fixed_name_reuse)
16
19
  end
17
20
 
18
21
  # 按清单顺序返回选中的设备。
@@ -26,7 +29,7 @@ module Net
26
29
  end
27
30
 
28
31
  selected_indices = {}
29
- filenames = {}
32
+ selected_filenames = {}
30
33
  previous_index = -1
31
34
  ready.each do |task|
32
35
  unless task.is_a?(Array) && task.size == 2 && task.first.is_a?(Integer)
@@ -39,9 +42,11 @@ module Net
39
42
  end
40
43
  if mode == :tftp
41
44
  filename = device.tftp_filename
42
- raise ArgumentError, "plan cannot upload multiple devices to one TFTP filename" if filenames.key?(filename)
45
+ if selected_filenames.key?(filename) && !(allow_fixed_name_reuse && device.vendor == :palo_alto)
46
+ raise ArgumentError, "plan cannot upload multiple devices to one TFTP filename"
47
+ end
43
48
 
44
- filenames[filename] = true
49
+ selected_filenames[filename] = true
45
50
  end
46
51
  selected_indices[index] = true
47
52
  previous_index = index
@@ -51,7 +56,7 @@ module Net
51
56
  result = outcomes[index]
52
57
  next if selected_indices[index] && result.nil?
53
58
 
54
- unless !selected_indices[index] && valid_skipped_outcome?(result, device, filenames)
59
+ unless !selected_indices[index] && valid_skipped_outcome?(result, device, selected_filenames)
55
60
  raise ArgumentError, "plan outcome does not match its inventory slot"
56
61
  end
57
62
  end
@@ -61,7 +66,7 @@ module Net
61
66
  private
62
67
 
63
68
  # 跳过结果不携带执行产物;文件冲突必须有本批次实际执行的目标作为依据。
64
- def valid_skipped_outcome?(result, device, filenames)
69
+ def valid_skipped_outcome?(result, device, selected_filenames)
65
70
  return false unless result.is_a?(Outcome) && result.device.equal?(device) && result.backup.nil? &&
66
71
  result.error_code.nil? && result.error_type.nil? &&
67
72
  result.started_at.nil? && result.finished_at.nil?
@@ -69,7 +74,7 @@ module Net
69
74
  return result.status == device.issue unless device.ready?
70
75
 
71
76
  result.status == :sample_limit ||
72
- (mode == :tftp && result.status == :remote_filename_collision && filenames.key?(device.tftp_filename))
77
+ (mode == :tftp && result.status == :remote_filename_collision && selected_filenames.key?(device.tftp_filename))
73
78
  end
74
79
  end
75
80
  end
@@ -13,21 +13,23 @@ module Net
13
13
  end
14
14
 
15
15
  # 生成本地或 TFTP 备份计划,明确记录每台未执行设备的原因。
16
- def call(mode:, limit_per_vendor:)
16
+ def call(mode:, limit_per_vendor:, allow_fixed_name_reuse: false)
17
17
  self.class.validate_limit!(limit_per_vendor)
18
- candidates = @inventory.each_with_index.select { |device, _index| device.ready? }
19
- selected = if limit_per_vendor
20
- candidates.group_by { |device, _index| device.vendor }.values.flat_map do |entries|
21
- entries.sort_by { |device, _index| device.host }.first(limit_per_vendor)
22
- end
23
- else
24
- candidates
25
- end
26
- selected_indices = selected.to_h { |_device, index| [index, true] }
27
- filenames = {}
18
+ ready_tasks = @inventory.each_with_index.filter_map do |device, index|
19
+ [index, device] if device.ready?
20
+ end
21
+ selected_tasks = if limit_per_vendor
22
+ ready_tasks.group_by { |_index, device| device.vendor }.values.flat_map do |tasks|
23
+ tasks.sort_by { |_index, device| device.host }.first(limit_per_vendor)
24
+ end
25
+ else
26
+ ready_tasks
27
+ end
28
+ selected_indices = selected_tasks.to_h { |index, _device| [index, true] }
29
+ first_index_by_filename = {}
28
30
  if mode == :tftp
29
31
  # 先按采样顺序保留每个目标的首台设备,再按清单顺序输出结果。
30
- selected.each { |device, index| filenames[device.tftp_filename] ||= index }
32
+ selected_tasks.each { |index, device| first_index_by_filename[device.tftp_filename] ||= index }
31
33
  end
32
34
  outcomes = Array.new(@inventory.size)
33
35
  ready = @inventory.each_with_index.filter_map do |device, index|
@@ -37,14 +39,16 @@ module Net
37
39
  elsif !selected_indices.key?(index)
38
40
  outcomes[index] = skipped(device, :sample_limit)
39
41
  nil
40
- elsif mode == :tftp && filenames.fetch(device.tftp_filename) != index
42
+ elsif mode == :tftp && first_index_by_filename.fetch(device.tftp_filename) != index &&
43
+ !(allow_fixed_name_reuse && device.vendor == :palo_alto)
41
44
  outcomes[index] = skipped(device, :remote_filename_collision)
42
45
  nil
43
46
  else
44
47
  [index, device].freeze
45
48
  end
46
49
  end
47
- Plan.new(mode: mode, inventory: @inventory, ready: ready.freeze, outcomes: outcomes.freeze)
50
+ Plan.new(mode: mode, inventory: @inventory, ready: ready.freeze, outcomes: outcomes.freeze,
51
+ allow_fixed_name_reuse: allow_fixed_name_reuse)
48
52
  end
49
53
 
50
54
  # 限制单厂商采样数,空值表示选择所有就绪设备。
@@ -0,0 +1,253 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "io/console"
4
+
5
+ module Net
6
+ module Connector
7
+ module Netdisco
8
+ # 共享一个实例接收并发会话事件和任务回调;人类进度不混入 JSON 输出。
9
+ class Progress
10
+ def initialize(io: $stderr, enabled: true, verbose: false)
11
+ @io, @enabled, @verbose = io, enabled, verbose
12
+ @tty = io.respond_to?(:tty?) && io.tty?
13
+ @stages = {}
14
+ @task_started = {}
15
+ @failures = Hash.new(0)
16
+ @started_at = Process.clock_gettime(Process::CLOCK_MONOTONIC)
17
+ @succeeded = @failed = @unverified = 0
18
+ @mutex = Mutex.new
19
+ @total = @completed = @active = 0
20
+ end
21
+
22
+ def reading_inventory
23
+ write(nil, "正在读取 Netdisco 设备清单")
24
+ end
25
+
26
+ def plan(plan, concurrency:, limit_per_vendor: nil)
27
+ @mutex.synchronize do
28
+ @total = plan.ready.size
29
+ @completed = @active = 0
30
+ @run_started = monotonic
31
+ mode = limit_per_vendor ? "抽样(每厂商最多 #{limit_per_vendor} 台)" : "全量"
32
+ emit(nil, "#{mode} | 清单 #{plan.inventory.size} 台;本次 #{@total} 台;跳过 #{plan.inventory.size - @total} 台;并发 #{concurrency}")
33
+ end
34
+ end
35
+
36
+ def start(device)
37
+ @mutex.synchronize do
38
+ @active += 1
39
+ @stages[device.host] = "登录"
40
+ @task_started[device.host] = monotonic
41
+ @verbose ? emit(device.host, "开始备份(#{device.vendor})") : refresh
42
+ end
43
+ end
44
+
45
+ # 只显示已脱敏事件中的阶段和安全命令,不显示设备回显或异常原文。
46
+ def event(event)
47
+ return unless event.is_a?(Log::Event)
48
+
49
+ unless @verbose
50
+ @mutex.synchronize do
51
+ host = event.fields[:host]
52
+ @stages[host] = "采集" if @stages.key?(host) && event.name == "login_complete" && event.fields[:status] == "ok"
53
+ refresh
54
+ end
55
+ return
56
+ end
57
+
58
+ fields = event.fields
59
+ message = case event.name
60
+ when "connect" then "正在连接并登录(#{fields[:protocol]})"
61
+ when "login_complete"
62
+ fields[:status] == "ok" ? "登录成功" : "登录失败(#{fields[:code]})"
63
+ when "connect_failed" then "连接失败(#{fields[:code]})"
64
+ when "command_start" then "执行命令:#{fields[:text]};等待响应"
65
+ when "command_complete"
66
+ fields[:status] == "response_received" ? "命令响应完成(#{fields[:duration_ms]} ms)" :
67
+ "命令失败(#{fields[:code]})"
68
+ when "operation_complete"
69
+ fields[:status] == "completed" ? "脚本处理完成" : "脚本处理失败(#{fields[:code]})"
70
+ end
71
+ write(fields[:host], message) if message
72
+ end
73
+
74
+ def result(outcome)
75
+ @mutex.synchronize do
76
+ @completed += 1
77
+ @active -= 1
78
+ @stages.delete(outcome.device.host)
79
+ @task_started.delete(outcome.device.host)
80
+ @failures[outcome.error_code || outcome.status] += 1 unless outcome.success?
81
+ outcome.success? ? @succeeded += 1 : @failed += 1
82
+ @unverified += 1 if outcome.status == :reported_uploaded && outcome.backup&.verification != :server_verified
83
+ message = case outcome.status
84
+ when :backed_up then "备份已保存(#{outcome.backup.bytes} 字节)"
85
+ when :reported_uploaded
86
+ outcome.backup&.verification == :server_verified ? "配置已归档(#{outcome.backup.server_bytes} 字节)" :
87
+ "设备报告上传完成;服务器文件尚未核验"
88
+ when :saved_with_error then "备份有产物但未完全成功(#{outcome.error_code})"
89
+ when :reported_with_error then "设备报告上传但收尾异常(#{outcome.error_code})"
90
+ else "未完成(#{outcome.error_code || outcome.status})"
91
+ end
92
+ hint = failure_hint(outcome.error_code)
93
+ message = "#{message};#{hint}" if hint
94
+ if @verbose || !outcome.success?
95
+ emit(outcome.device.host, "#{message};耗时 #{outcome.duration_ms} ms")
96
+ elsif !@tty && ((@completed % 25).zero? || @completed == @total)
97
+ emit(nil, status_line)
98
+ end
99
+ refresh
100
+ end
101
+ end
102
+
103
+ def finish(report)
104
+ summary = report.summary
105
+ failures = report.outcomes.reject { |item| item.success? || %i[filtered sample_limit].include?(item.status) }
106
+ .group_by { |item| item.error_code || item.status }.transform_values(&:size)
107
+ unless failures.empty?
108
+ write(nil, "未完成分类:#{failures.sort_by { |code, count| [-count, code.to_s] }.map { |code, count| "#{code}=#{count}" }.join(",")}")
109
+ end
110
+ write(nil, "批次结束:成功 #{summary[:succeeded]},部分成功 #{summary[:partial]},失败 #{summary[:failed]},跳过 #{summary[:skipped]};耗时 #{(Process.clock_gettime(Process::CLOCK_MONOTONIC) - @started_at).round(1)} 秒;结果 #{report.policy_success? ? "完成" : "未完成"}")
111
+ unverified = summary[:verification]&.fetch(:unverified, 0) || 0
112
+ write(nil, "其中 #{unverified} 台仅设备报告上传完成,服务器文件尚未核验") if unverified.positive?
113
+ return unless !report.callback_errors.empty? || report.report_error
114
+
115
+ write(nil, "回调异常 #{report.callback_errors.size},报告错误 #{report.report_error || "无"}")
116
+ end
117
+
118
+ # 心跳只负责显示;退出、异常或中断时先唤醒并回收线程。
119
+ def with_updates
120
+ return yield unless @enabled && @tty && !@verbose
121
+
122
+ lock = Mutex.new
123
+ signal = ConditionVariable.new
124
+ stopped = false
125
+ worker = Thread.new do
126
+ loop do
127
+ stop = lock.synchronize do
128
+ signal.wait(lock, 1) unless stopped
129
+ stopped
130
+ end
131
+ break if stop
132
+
133
+ tick
134
+ end
135
+ end
136
+ yield
137
+ ensure
138
+ if worker
139
+ lock.synchronize { stopped = true; signal.broadcast }
140
+ worker.value
141
+ @mutex.synchronize { clear_refresh }
142
+ end
143
+ end
144
+
145
+ def tick
146
+ @mutex.synchronize { refresh(force: true) }
147
+ end
148
+
149
+ def location(path)
150
+ write(nil, "报告:#{path}")
151
+ end
152
+
153
+ private
154
+
155
+ def failure_hint(code)
156
+ case code
157
+ when :authentication_failed then "设备拒绝认证,请核对账号、密码或 AAA 策略"
158
+ when :authentication_error then "认证交互未完成,请核对认证方式或设备 AAA 日志"
159
+ when :host_key_untrusted then "主机密钥尚未信任,请确认指纹并登记"
160
+ when :host_key_changed then "已登记主机密钥发生变化,请核实设备身份"
161
+ when :login_timeout then "未在期限内获得登录提示,请检查 SSH 握手与登录交互"
162
+ when :connection_timeout then "SSH 连接或握手超时,请检查服务与网络"
163
+ when :device_error then "设备拒绝命令,请核对厂商映射和账号权限"
164
+ end
165
+ end
166
+
167
+ def status_line
168
+ percent = @total.positive? ? @completed * 100 / @total : 0
169
+ "#{@completed}/#{@total} #{percent}% | 成功 #{@succeeded} 未完成 #{@failed} | 登录 #{@stages.values.count("登录")} 采集 #{@stages.values.count("采集")} 排队 #{[@total - @completed - @active, 0].max}"
170
+ end
171
+
172
+ def monotonic = Process.clock_gettime(Process::CLOCK_MONOTONIC)
173
+
174
+ def duration(seconds)
175
+ total = seconds.round
176
+ format("%02d:%02d:%02d", total / 3600, total / 60 % 60, total % 60)
177
+ end
178
+
179
+ def timing_line
180
+ elapsed = [monotonic - (@run_started || @started_at), 0].max
181
+ rate = elapsed.positive? ? @completed / elapsed : 0
182
+ eta = if @completed == @total && @total.positive?
183
+ "00:00:00"
184
+ elsif @completed >= 5 && rate.positive?
185
+ "约 #{duration((@total - @completed) / rate)}"
186
+ else
187
+ "估算中"
188
+ end
189
+ line = "耗时 #{duration(elapsed)} | #{format("%.1f", rate)} 台/秒 | 剩余 #{eta}"
190
+ host, started = @task_started.min_by { |_key, value| value }
191
+ if started && monotonic - started >= 10
192
+ line += " | 最久 #{host} #{@stages[host]} #{(monotonic - started).floor}s"
193
+ end
194
+ line
195
+ end
196
+
197
+ def refresh(force: false)
198
+ return unless @enabled && @tty && !@verbose
199
+
200
+ now = monotonic
201
+ return if !force && @refreshing && @completed < @total && @last_refresh && now - @last_refresh < 0.2
202
+
203
+ @last_refresh = now
204
+ divider = fit_terminal("-" * 100)
205
+ lines = [divider, fit_terminal(status_line), fit_terminal(timing_line), divider]
206
+ return if @refreshing && @last_status == lines
207
+
208
+ clear_refresh
209
+ @last_status = lines
210
+ @io.write(lines.join("\n"))
211
+ @io.flush
212
+ @refreshing = true
213
+ end
214
+
215
+ # 留一列避免终端自动换行,窄窗口也不会留下旧进度行。
216
+ def fit_terminal(line)
217
+ columns = @io.respond_to?(:winsize) ? @io.winsize.last : 120
218
+ columns = 120 unless columns.positive?
219
+ width = 0
220
+ text = line.gsub(/[[:cntrl:]]/, " ")
221
+ text.each_char.take_while do |character|
222
+ width += character.ord > 255 ? 2 : 1
223
+ width < columns
224
+ end.join
225
+ rescue IOError, SystemCallError
226
+ line
227
+ end
228
+
229
+ def clear_refresh
230
+ return unless @refreshing
231
+
232
+ @io.write("\r\e[2K" + ("\e[1A\r\e[2K" * (@last_status.length - 1)))
233
+ @refreshing = false
234
+ end
235
+
236
+ def write(host, message)
237
+ @mutex.synchronize { emit(host, message) }
238
+ end
239
+
240
+ def emit(host, message)
241
+ return unless @enabled
242
+
243
+ percent = @total.positive? ? " #{(@completed * 100.0 / @total).floor}%" : ""
244
+ clear_refresh
245
+ line = "[#{Time.now.strftime("%H:%M:%S")}] [#{@completed}/#{@total}#{percent} 执行中 #{@active}] [#{host || "批次"}] #{message}"
246
+ line = "[#{Time.now.strftime("%H:%M:%S")}] #{host ? "#{host} " : ""}#{message}" unless @verbose
247
+ @io.write(line.gsub(/[[:cntrl:]]/, " ") + "\n")
248
+ @io.flush
249
+ end
250
+ end
251
+ end
252
+ end
253
+ end