net-connector 0.9.0 → 0.10.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (37) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +9 -1
  3. data/docs/VERIFICATION.md +1 -1
  4. data/docs/architecture.md +16 -5
  5. data/docs/netdisco.md +54 -1
  6. data/docs/netdisco.zh-CN.md +46 -2
  7. data/docs/usage.md +24 -7
  8. data/docs/usage.zh-CN.md +22 -8
  9. data/examples/backup.yml +1 -1
  10. data/lib/net/connector/background_task.rb +69 -0
  11. data/lib/net/connector/device/topology/interface_description.rb +3 -3
  12. data/lib/net/connector/device/topology/interface_name.rb +10 -19
  13. data/lib/net/connector/device/topology/stages.rb +50 -0
  14. data/lib/net/connector/device/topology.rb +65 -24
  15. data/lib/net/connector/device_session.rb +38 -0
  16. data/lib/net/connector/engine/transport.rb +15 -5
  17. data/lib/net/connector/execution_pool.rb +108 -0
  18. data/lib/net/connector/netdisco/backup_run.rb +14 -2
  19. data/lib/net/connector/netdisco/cli/options.rb +1 -0
  20. data/lib/net/connector/netdisco/cli.rb +18 -7
  21. data/lib/net/connector/netdisco/client.rb +5 -6
  22. data/lib/net/connector/netdisco/config_file.rb +5 -2
  23. data/lib/net/connector/netdisco/database_client.rb +33 -15
  24. data/lib/net/connector/netdisco/fleet.rb +68 -36
  25. data/lib/net/connector/netdisco/inventory_cache.rb +218 -0
  26. data/lib/net/connector/netdisco/operation_batch.rb +154 -0
  27. data/lib/net/connector/netdisco/operations.rb +268 -0
  28. data/lib/net/connector/netdisco/planner.rb +26 -17
  29. data/lib/net/connector/netdisco/progress.rb +22 -10
  30. data/lib/net/connector/netdisco/settings.rb +36 -6
  31. data/lib/net/connector/netdisco/tftp_archive.rb +2 -1
  32. data/lib/net/connector/netdisco/tftp_verification.rb +36 -11
  33. data/lib/net/connector/netdisco/worker.rb +7 -46
  34. data/lib/net/connector/vendor/h3c/topology.rb +1 -1
  35. data/lib/net/connector/vendor/hillstone/topology.rb +1 -1
  36. data/lib/net/connector/version.rb +1 -1
  37. metadata +8 -1
@@ -0,0 +1,268 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "ipaddr"
4
+ require "time"
5
+ require "fileutils"
6
+ require_relative "../execution_pool"
7
+ require_relative "../device_session"
8
+ require_relative "../device/topology"
9
+ require_relative "settings"
10
+ require_relative "operation_batch"
11
+
12
+ module Net
13
+ module Connector
14
+ module Netdisco
15
+ # 配置执行与接口修正共用设备生命周期,各业务仍保留自己的审批和结果。
16
+ class Operations
17
+ DEFAULT_SETTING = Object.new.freeze
18
+ MissingCredentials = Class.new(StandardError)
19
+ private_constant :DEFAULT_SETTING, :MissingCredentials
20
+
21
+ def initialize(settings:, inventory:, credentials: nil, connector_factory: nil)
22
+ raise ArgumentError, "inventory must respond to call" unless inventory.respond_to?(:call)
23
+ raise ArgumentError, "credentials must respond to call" unless credentials.nil? || credentials.respond_to?(:call)
24
+ unless connector_factory.nil? || connector_factory.respond_to?(:call)
25
+ raise ArgumentError, "connector_factory must respond to call"
26
+ end
27
+
28
+ @settings, @inventory, @credentials = settings, inventory, credentials
29
+ @connector_factory = connector_factory || ->(device, options) { device.build_connector(**options) }
30
+ end
31
+
32
+ # 全批脚本先解析并固定,明确确认后才查询清单和读取逐设备凭据。
33
+ def execute_scripts(scripts:, confirmed: false, concurrency: DEFAULT_SETTING, on_start: nil, on_result: nil)
34
+ validate_confirmation!(confirmed)
35
+ tasks = normalize_mapping(scripts, :scripts) do |script|
36
+ script.is_a?(String) ? Script.parse(script) : (script.is_a?(Script) ? script : Script.new(script))
37
+ end
38
+ run(:execute_scripts, tasks, concurrency: concurrency, on_start: on_start, on_result: on_result) do |connector, script|
39
+ result = connector.execute_script(script)
40
+ raise TypeError, "configuration operation must return a Result" unless result.is_a?(Result)
41
+
42
+ result
43
+ end
44
+ end
45
+
46
+ # 一次调用完成邻居发现与描述同步;每台设备独占会话,普通失败互不影响。
47
+ def sync_interface_descriptions(hosts:, abbreviate: true, lowercase: false, concurrency: DEFAULT_SETTING,
48
+ on_start: nil, on_result: nil, &formatter)
49
+ tasks = description_targets(hosts, abbreviate, lowercase)
50
+ run(:sync_interface_descriptions, tasks, concurrency: concurrency, on_start: on_start, on_result: on_result) do |connector, _|
51
+ result = connector.sync_interface_descriptions(abbreviate: abbreviate, lowercase: lowercase, &formatter)
52
+ raise TypeError, "description synchronization must return a Result" unless result.is_a?(Result)
53
+
54
+ result
55
+ end
56
+ end
57
+
58
+ # 规划只读取现场证据;格式化回调并发执行,不承担设备连接或审批职责。
59
+ def plan_interface_descriptions(hosts:, abbreviate: true, lowercase: false, concurrency: DEFAULT_SETTING,
60
+ on_start: nil, on_result: nil, &formatter)
61
+ tasks = description_targets(hosts, abbreviate, lowercase)
62
+
63
+ run(:plan_interface_descriptions, tasks, concurrency: concurrency, on_start: on_start, on_result: on_result) do |connector, _|
64
+ result = connector.plan_interface_descriptions(abbreviate: abbreviate, lowercase: lowercase, &formatter)
65
+ raise TypeError, "description planning must return a Topology::Plan" unless result.is_a?(Topology::Plan)
66
+
67
+ copy_plan(result)
68
+ end
69
+ end
70
+
71
+ # 审批计划整体复制并验证;单设备应用仍执行现场重验、读回和保存,不直接重放 commands。
72
+ def apply_interface_descriptions(plans:, confirmed: false, concurrency: DEFAULT_SETTING, on_start: nil, on_result: nil)
73
+ validate_confirmation!(confirmed)
74
+ tasks = normalize_mapping(plans, :plans) { |plan| copy_plan(plan) }
75
+ tasks.each do |host, plan|
76
+ raise ArgumentError, "plan host does not match its target" unless normalize_host(plan.host) == host
77
+ end
78
+ run(:apply_interface_descriptions, tasks, concurrency: concurrency, on_start: on_start, on_result: on_result) do |connector, plan|
79
+ result = connector.apply_interface_descriptions(plan, confirmed: true)
80
+ raise TypeError, "description application must return a Result" unless result.is_a?(Result)
81
+
82
+ result
83
+ end
84
+ end
85
+
86
+ private
87
+
88
+ # 规划与同步共用目标校验,整批输入在清单与连接之前固定。
89
+ def description_targets(hosts, abbreviate, lowercase)
90
+ raise ArgumentError, "hosts must be an Array" unless hosts.is_a?(Array)
91
+ unless [abbreviate, lowercase].all? { |value| value == true || value == false }
92
+ raise ArgumentError, "description formatting options must be boolean"
93
+ end
94
+
95
+ tasks = normalize_mapping(hosts.to_h { |host| [host, nil] }, :hosts)
96
+ raise ArgumentError, "hosts must be unique" unless tasks.size == hosts.size
97
+
98
+ tasks
99
+ end
100
+
101
+ def validate_confirmation!(confirmed)
102
+ raise ArgumentError, "configuration changes require confirmed: true" unless confirmed == true
103
+ end
104
+
105
+ # 管理地址规范化之后再检查重复,等价的 IPv6 写法不能创建两个并行写会话。
106
+ def normalize_host(host)
107
+ unless host.is_a?(String) && !host.empty? && !host.include?("/")
108
+ raise ArgumentError, "targets must be IP address Strings without a prefix"
109
+ end
110
+
111
+ IPAddr.new(host).to_s.freeze
112
+ rescue IPAddr::Error
113
+ raise ArgumentError, "target contains an invalid IP address", cause: nil
114
+ end
115
+
116
+ def normalize_mapping(mapping, name)
117
+ raise ArgumentError, "#{name} must be a Hash" unless mapping.is_a?(Hash)
118
+
119
+ mapping.each_with_object({}) do |(host, value), tasks|
120
+ host = normalize_host(host)
121
+ raise ArgumentError, "targets must be unique" if tasks.key?(host)
122
+
123
+ tasks[host] = block_given? ? yield(value) : value
124
+ end.freeze
125
+ end
126
+
127
+ # 外部审批容器不与执行共享可变子对象,非法领域结构在任何连接前拒绝。
128
+ def copy_plan(plan)
129
+ raise ArgumentError, "plans must contain Topology::Plan values" unless plan.is_a?(Topology::Plan)
130
+ normalize_host(plan.host)
131
+ unless Net::Connector.vendors.include?(plan.vendor) && plan.evidence.is_a?(Hash) &&
132
+ plan.changes.is_a?(Array) && plan.commands.is_a?(Array) && plan.commands.all?(String)
133
+ raise ArgumentError, "description plan contains invalid fields"
134
+ end
135
+ commands = Script.new(plan.commands).map { |command| command.text.dup.freeze }.freeze
136
+ evidence = plan.evidence.to_h do |interface, fields|
137
+ unless interface.is_a?(String) && fields.is_a?(Array) && fields.size == 3 && fields.all?(String)
138
+ raise ArgumentError, "description plan contains invalid evidence"
139
+ end
140
+ [interface.dup.freeze, fields.map { |field| field.dup.freeze }.freeze]
141
+ end.freeze
142
+ changes = plan.changes.map { |change| copy_change(change) }.freeze
143
+ raise ArgumentError, "empty description plan cannot contain commands" if changes.empty? && !commands.empty?
144
+
145
+ Topology::Plan.new(host: normalize_host(plan.host), vendor: plan.vendor, evidence: evidence,
146
+ changes: changes, commands: commands).freeze
147
+ end
148
+
149
+ def copy_change(change)
150
+ unless change.is_a?(Topology::Change) && change.old_description.is_a?(String) &&
151
+ change.neighbor.is_a?(Topology::Neighbor) && change.neighbor.to_h.values.all? { |value| value.is_a?(String) || value.is_a?(Symbol) }
152
+ raise ArgumentError, "description plan contains an invalid change"
153
+ end
154
+ Topology::InterfaceDescription.validate_interface!(change.interface)
155
+ Topology::InterfaceDescription.validate!(change.new_description)
156
+ neighbor = Topology::Neighbor.new(**change.neighbor.to_h.transform_values { |value| value.is_a?(String) ? value.dup.freeze : value }).freeze
157
+ Topology::Change.new(interface: change.interface.dup.freeze, old_description: change.old_description.dup.freeze,
158
+ new_description: change.new_description.dup.freeze, neighbor: neighbor).freeze
159
+ end
160
+
161
+ # 策略只固定一次;清单与连接使用同一快照,凭据仍在每台任务开始时动态读取。
162
+ def run(operation, tasks, concurrency:, on_start:, on_result:)
163
+ [on_start, on_result].each do |callback|
164
+ raise ArgumentError, "callbacks must respond to call" unless callback.nil? || callback.respond_to?(:call)
165
+ end
166
+ ExecutionPool.validate_concurrency!(concurrency) unless concurrency.equal?(DEFAULT_SETTING)
167
+ snapshot = @settings.snapshot(mode: :inventory)
168
+ concurrency = snapshot.concurrency if concurrency.equal?(DEFAULT_SETTING)
169
+ ExecutionPool.validate_concurrency!(concurrency)
170
+ return empty_batch(operation) if tasks.empty?
171
+
172
+ devices = select_devices(tasks, snapshot)
173
+ FileUtils.mkdir_p(snapshot.log_directory, mode: 0o700) if snapshot.log_directory
174
+ started_at = Time.now.utc
175
+ callback_errors = Queue.new
176
+ outcomes = ExecutionPool.new(concurrency: concurrency).map(devices) do |device|
177
+ perform(device, snapshot, callback_errors, on_start, on_result) { |connector| yield connector, tasks.fetch(device.host) }
178
+ end
179
+ errors = Array.new(callback_errors.size) { callback_errors.pop }
180
+ OperationBatch.new(operation: operation, outcomes: outcomes, callback_errors: errors,
181
+ started_at: started_at, finished_at: Time.now.utc)
182
+ end
183
+
184
+ def empty_batch(operation)
185
+ now = Time.now.utc
186
+ OperationBatch.new(operation: operation, outcomes: [], callback_errors: [], started_at: now, finished_at: now)
187
+ end
188
+
189
+ # 所有目标必须唯一、可执行且厂商匹配;后面的无效目标不能在前面设备已写后才发现。
190
+ def select_devices(tasks, snapshot)
191
+ inventory = @inventory.call(snapshot)
192
+ raise ArgumentError, "inventory must return an Array of Devices" unless inventory.is_a?(Array) && inventory.all?(Device)
193
+
194
+ by_host = {}
195
+ duplicates = {}
196
+ inventory.each do |device|
197
+ next unless tasks.key?(device.host)
198
+
199
+ duplicates[device.host] = true if by_host.key?(device.host)
200
+ by_host[device.host] = device
201
+ end
202
+ tasks.map do |host, value|
203
+ device = by_host[host]
204
+ unless device&.ready? && !duplicates.key?(host)
205
+ raise ArgumentError, "target must identify exactly one ready inventory device"
206
+ end
207
+ if value.is_a?(Topology::Plan)
208
+ Topology.validate_plan!(value, host: device.host, vendor: device.vendor)
209
+ elsif value.nil?
210
+ Topology.validate_vendor_support!(host: device.host, vendor: device.vendor, phase: :plan)
211
+ end
212
+ device
213
+ end
214
+ end
215
+
216
+ # 返回失败的 Result 仍携带已完成步骤;设备关闭故障也不能覆盖原操作错误。
217
+ def perform(device, snapshot, callback_errors, on_start, on_result, &operation)
218
+ started_at = Time.now.utc
219
+ started = Process.clock_gettime(Process::CLOCK_MONOTONIC)
220
+ notify(on_start, device, callback_errors, device.host)
221
+ value, error = DeviceSession.call(factory: -> { build_connector(device, snapshot) }) do |connector|
222
+ result = operation.call(connector)
223
+ if result.is_a?(Topology::Plan) && (result.host != device.host || result.vendor != device.vendor)
224
+ raise ArgumentError, "planned evidence does not match its inventory device"
225
+ end
226
+ result
227
+ end
228
+ error = value.error || error if value.is_a?(Result)
229
+ status = if error.is_a?(MissingCredentials)
230
+ error = nil
231
+ :missing_credentials
232
+ elsif error
233
+ value && (!value.is_a?(Result) || value.success? || !value.steps.empty?) ? :partial : :failed
234
+ else
235
+ :succeeded
236
+ end
237
+ outcome = OperationOutcome.new(device: device, status: status, value: value, error: error,
238
+ started_at: started_at, finished_at: Time.now.utc,
239
+ duration_ms: ((Process.clock_gettime(Process::CLOCK_MONOTONIC) - started) * 1000).round)
240
+ notify(on_result, outcome, callback_errors, device.host)
241
+ outcome
242
+ end
243
+
244
+ def build_connector(device, snapshot)
245
+ options = @credentials ? @credentials.call(device) : @settings.device_credentials_for(device)
246
+ raise MissingCredentials if options.nil?
247
+
248
+ unless options.is_a?(Hash) && options.keys.all?(Symbol) && !options.key?(:host) &&
249
+ options[:username].is_a?(String) && !options[:username].empty?
250
+ raise ArgumentError, "credential resolver must return settings with username and without host"
251
+ end
252
+ options = snapshot.connection_options(device.vendor).merge(options)
253
+ if snapshot.log_directory
254
+ options[:log_file] = File.join(snapshot.log_directory, "#{device.host.tr(":", "_")}.log")
255
+ end
256
+ @connector_factory.call(device, options)
257
+ end
258
+
259
+ # 普通通知故障只保存白名单类型;中断仍由线程池回收整批设备会话。
260
+ def notify(callback, value, errors, host)
261
+ callback&.call(value)
262
+ rescue StandardError => error
263
+ errors << { host: host, error_type: Diagnostic.from(error).error_type }.freeze
264
+ end
265
+ end
266
+ end
267
+ end
268
+ end
@@ -16,28 +16,16 @@ module Net
16
16
  # 生成本地或 TFTP 备份计划,明确记录每台未执行设备的原因。
17
17
  def call(mode:, limit_per_vendor:, allow_fixed_name_reuse: false)
18
18
  self.class.validate_limit!(limit_per_vendor)
19
- ready_tasks = @inventory.each_with_index.filter_map do |device, index|
20
- [index, device] if device.ready?
21
- end
22
- selected_tasks = if limit_per_vendor
23
- ready_tasks.group_by { |_index, device| device.vendor }.values.flat_map do |tasks|
24
- tasks.min_by(limit_per_vendor) { |_index, device| device.host }
25
- end
26
- else
27
- ready_tasks
28
- end
29
- selected_indices = selected_tasks.to_h { |index, _device| [index, true] }
30
- first_index_by_filename = {}
31
- if mode == :tftp
32
- # 先按采样顺序保留每个目标的首台设备,再按清单顺序输出结果。
33
- selected_tasks.each { |index, device| first_index_by_filename[device.tftp_filename] ||= index }
34
- end
19
+ # 默认全量不构造采样任务副本或逐设备选择索引;只保留最终计划需要的槽位。
20
+ selected_tasks = sample_tasks(limit_per_vendor)
21
+ selected_indices = selected_tasks&.to_h { |index, _device| [index, true] }
22
+ first_index_by_filename = mode == :tftp ? tftp_target_indices(selected_tasks) : {}
35
23
  outcomes = Array.new(@inventory.size)
36
24
  ready = @inventory.each_with_index.filter_map do |device, index|
37
25
  if !device.ready?
38
26
  outcomes[index] = skipped(device, device.issue)
39
27
  nil
40
- elsif !selected_indices.key?(index)
28
+ elsif selected_indices && !selected_indices.key?(index)
41
29
  outcomes[index] = skipped(device, :sample_limit)
42
30
  nil
43
31
  elsif mode == :tftp && first_index_by_filename.fetch(device.tftp_filename) != index &&
@@ -61,6 +49,27 @@ module Net
61
49
 
62
50
  private
63
51
 
52
+ # 只有显式采样才分组;厂商组内仍按管理地址选择,与全量输出顺序分开。
53
+ def sample_tasks(limit)
54
+ return unless limit
55
+
56
+ tasks = @inventory.each_with_index.filter_map { |device, index| [index, device] if device.ready? }
57
+ tasks.group_by { |_index, device| device.vendor }.values.flat_map do |group|
58
+ group.min_by(limit) { |_index, device| device.host }
59
+ end
60
+ end
61
+
62
+ # 采样模式按采样顺序仲裁目标,全量模式直接扫描清单;输出仍保留清单顺序。
63
+ def tftp_target_indices(selected_tasks)
64
+ found = {}
65
+ if selected_tasks
66
+ selected_tasks.each { |index, device| found[device.tftp_filename] ||= index }
67
+ else
68
+ @inventory.each_with_index { |device, index| found[device.tftp_filename] ||= index if device.ready? }
69
+ end
70
+ found
71
+ end
72
+
64
73
  # 构建计划阶段无需设备连接的跳过结果。
65
74
  def skipped(device, status)
66
75
  Outcome.new(device: device, status: status, backup: nil, error_code: nil, error_type: nil)
@@ -3,6 +3,7 @@
3
3
  require "io/console"
4
4
  require_relative "../engine/error_metadata"
5
5
  require_relative "../engine/log/event"
6
+ require_relative "../background_task"
6
7
 
7
8
  module Net
8
9
  module Connector
@@ -127,13 +128,13 @@ module Net
127
128
  end
128
129
 
129
130
  # 心跳只负责显示;退出、异常或中断时先唤醒并回收线程。
130
- def with_updates
131
+ def with_updates(&)
131
132
  return yield unless @enabled && @tty && !@verbose
132
133
 
133
134
  lock = Mutex.new
134
135
  signal = ConditionVariable.new
135
136
  stopped = false
136
- worker = Thread.new do
137
+ task = lambda do
137
138
  loop do
138
139
  stop = lock.synchronize do
139
140
  signal.wait(lock, 1) unless stopped
@@ -143,13 +144,20 @@ module Net
143
144
 
144
145
  tick
145
146
  end
147
+ rescue StandardError => error
148
+ # 非 IO 的显示故障也只停用进度;完整报告仍记录这一受控诊断。
149
+ @mutex.synchronize { record_output_error(error) }
146
150
  end
147
- yield
151
+ stop = -> { lock.synchronize { stopped = true; signal.broadcast } }
152
+ BackgroundTask.run(task: task, stop: stop, &)
153
+ rescue Exception => error # rubocop:disable Lint/RescueException -- 后台已回收,最后清屏也不能覆盖原业务故障或取消。
154
+ primary_failure = error
155
+ raise
148
156
  ensure
149
- if worker
150
- lock.synchronize { stopped = true; signal.broadcast }
151
- worker.value
152
- @mutex.synchronize { clear_refresh }
157
+ begin
158
+ Thread.handle_interrupt(Exception => :never) { @mutex.synchronize { clear_refresh } if task }
159
+ rescue Exception => error # rubocop:disable Lint/RescueException -- 清屏中的取消正常传播,已有主故障时保持其身份。
160
+ raise error unless primary_failure
153
161
  end
154
162
  end
155
163
 
@@ -268,11 +276,15 @@ module Net
268
276
  @io.write(text)
269
277
  @io.flush
270
278
  true
271
- rescue IOError, SystemCallError => error
272
- @output_error = { host: nil, error_type: ErrorMetadata.type(error.class.name) }.freeze
273
- @enabled = false
279
+ rescue StandardError => error
280
+ record_output_error(error)
274
281
  false
275
282
  end
283
+
284
+ def record_output_error(error)
285
+ @output_error = { host: nil, error_type: ErrorMetadata.type(error.class.name) || "StandardError" }.freeze
286
+ @enabled = false
287
+ end
276
288
  end
277
289
  end
278
290
  end
@@ -1,9 +1,12 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  require "json"
4
+ require "digest"
4
5
  require_relative "../../connector"
6
+ require_relative "../execution_pool"
5
7
  require_relative "client"
6
8
  require_relative "database_client"
9
+ require_relative "inventory_cache"
7
10
  require_relative "config_file"
8
11
  require_relative "rules"
9
12
  require_relative "planner"
@@ -14,10 +17,10 @@ module Net
14
17
  # 活的凭据来源与不可变的批次策略分开保存;只有明确列出的非敏感字段进入快照。
15
18
  class Settings
16
19
  DEFAULT_BACKUP_DIRECTORY = "backups"
17
- DEFAULT_CONCURRENCY = 4
18
- MAX_CONCURRENCY = 50
20
+ DEFAULT_CONCURRENCY = ExecutionPool::DEFAULT_CONCURRENCY
21
+ MAX_CONCURRENCY = ExecutionPool::MAX_CONCURRENCY
19
22
  # 这里只列可从环境读取的公开策略;凭据单独动态读取,其他内部键只供 YAML/CLI 使用。
20
- ENVIRONMENT_KEYS = %w[ND_URL].freeze
23
+ ENVIRONMENT_KEYS = %w[ND_URL ND_CACHE_TTL NC_CONCURRENCY].freeze
21
24
  DEFAULT_PAGE_SIZE = Client::DEFAULTS.fetch(:page_size)
22
25
  UNSET = Object.new.freeze
23
26
  private_constant :UNSET
@@ -80,6 +83,7 @@ module Net
80
83
  return self if mode == :export
81
84
 
82
85
  client_options
86
+ inventory_cache_options
83
87
  rules
84
88
  concurrency
85
89
  limit_per_vendor
@@ -107,7 +111,9 @@ module Net
107
111
  values[key] = item if item
108
112
  end
109
113
  connection.merge!(user: credentials[:username], password: credentials[:password]) if credentials
110
- return DatabaseClient.new(connection_options: connection, **options)
114
+ client = DatabaseClient.new(connection_options: connection, **options)
115
+ identity = [:postgres, connection.except(:password), options.slice(:query, :query_params)]
116
+ return cached_client(client, policy, identity)
111
117
  end
112
118
  raise ArgumentError, "ND_URL is required" unless options.fetch(:url)
113
119
 
@@ -118,7 +124,9 @@ module Net
118
124
  else
119
125
  { username: required_secret("ND_USERNAME"), password: required_secret("ND_PASSWORD") }
120
126
  end
121
- Client.new(**options, **authentication)
127
+ client = Client.new(**options, **authentication)
128
+ identity = [:http, Client.validate_url!(options.fetch(:url)).normalize.to_s, authentication.except(:password)]
129
+ cached_client(client, policy, identity)
122
130
  end
123
131
 
124
132
  # 清单来源来自 YAML/CLI,HTTP 与 PostgreSQL 共用后续计划和执行接口。
@@ -136,6 +144,16 @@ module Net
136
144
  inventory_source == :postgres ? postgres_options(options) : http_options(options)
137
145
  end
138
146
 
147
+ # 默认三天,可由环境覆盖;缓存策略随批次固定,不缓存设备凭据或业务计划。
148
+ def inventory_cache_options
149
+ directory = raw("ND_CACHE_DIRECTORY", ".cache/net-connector/inventory")
150
+ unless directory.is_a?(String) && !directory.empty? && !directory.include?("\0")
151
+ raise ArgumentError, "netdisco.cache_directory must be a nonempty path without NUL"
152
+ end
153
+ { directory: File.expand_path(directory), ttl: number("ND_CACHE_TTL", InventoryCache::DEFAULT_TTL),
154
+ enabled: boolean("ND_CACHE_ENABLED", true) }.freeze
155
+ end
156
+
139
157
  # 将筛选和映射策略交给 Rules 校验,未识别设备由清单结果单独标记。
140
158
  def rules
141
159
  Rules.new(include_hosts: list("NC_INCLUDE_HOSTS"),
@@ -278,7 +296,7 @@ module Net
278
296
  # 只输出公开策略;数据库身份和设备凭据不得通过配置摘要进入报告。
279
297
  def config_hash
280
298
  {
281
- netdisco: client_options.merge(source: inventory_source, database: database_options),
299
+ netdisco: client_options.merge(source: inventory_source, database: database_options, cache: inventory_cache_options),
282
300
  backup: { directory: backup_directory, concurrency: concurrency, limit_per_vendor: limit_per_vendor },
283
301
  inventory: { include_hosts: list("NC_INCLUDE_HOSTS"),
284
302
  exclude_hosts: list("NC_EXCLUDE_HOSTS"),
@@ -310,6 +328,18 @@ module Net
310
328
 
311
329
  private
312
330
 
331
+ # 身份指纹只用于隔离缓存;凭据、SQL 和参数均不写入缓存文件或摘要。
332
+ def cached_client(client, policy, identity)
333
+ cache = policy.inventory_cache_options
334
+ return client unless cache.fetch(:enabled)
335
+
336
+ options = policy.client_options
337
+ InventoryCache.new(client: client, **cache.except(:enabled),
338
+ identity: Digest::SHA256.hexdigest(JSON.generate(identity)),
339
+ **options.slice(:max_inventory_bytes, :max_devices),
340
+ lock_timeout: [5, options.fetch(:inventory_timeout)].min)
341
+ end
342
+
313
343
  def client_budget_options(defaults)
314
344
  defaults.to_h do |name, default|
315
345
  key = "ND_#{name.to_s.upcase}"
@@ -68,7 +68,8 @@ module Net
68
68
  receipt = outcome.backup
69
69
  return outcome unless receipt.is_a?(TftpReceipt) && receipt.path
70
70
 
71
- verified = @verifier.verify_outcome(outcome.with(started_at: started_at), started_at: upload_started_at, previous: previous)
71
+ verified = @verifier.verify_outcome(outcome.with(started_at: started_at), started_at: upload_started_at,
72
+ previous: previous, expected_path: remote_filename)
72
73
  return outcome.success? ? archive_failure(outcome) : outcome unless verified.backup.verification == :server_verified
73
74
 
74
75
  archive_uploaded_file(verified)
@@ -1,7 +1,10 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  require_relative "../storage/safe_file"
4
+ require_relative "../storage/backup_lock"
4
5
  require_relative "../device/tftp/receipt"
6
+ require_relative "diagnostic"
7
+ require "tempfile"
5
8
 
6
9
  module Net
7
10
  module Connector
@@ -12,31 +15,53 @@ module Net
12
15
  @root = root && File.realpath(root)
13
16
  end
14
17
 
15
- def call(report)
16
- return report unless @root
18
+ # 基线、设备上传和核验共用目标锁;只有上传后变化的文件才属于本次任务。
19
+ def upload_and_verify(device, path: device.tftp_filename)
20
+ outcome = nil
21
+ raise ArgumentError, "upload requires a block" unless block_given?
17
22
 
18
- outcomes = report.outcomes.map do |outcome|
19
- receipt = outcome.backup
20
- next outcome unless receipt.is_a?(TftpReceipt) && receipt.path && outcome.started_at
23
+ TftpTarget.validate_path!(path)
24
+ return yield path unless @root
21
25
 
22
- verify(outcome, receipt)
26
+ destination = server_path(path)
27
+ Storage::BackupLock.synchronize(destination, host: device.host, timeout: 240) do
28
+ previous = Storage::SafeFile.open(destination, missing: true) { |_file, stat| stat }
29
+ started_at = Tempfile.create(".net-connector-upload-", @root) { |file| file.stat.mtime }
30
+ outcome = yield path
31
+ outcome = verify_outcome(outcome, started_at: started_at, previous: previous, expected_path: path)
23
32
  end
24
- report.with(outcomes: outcomes.freeze)
33
+ rescue StandardError => error
34
+ raise unless outcome
35
+ return outcome unless outcome.success?
36
+
37
+ # 上传后的核验或解锁故障保留回执,不能让 Worker 把已上传结果退回普通失败。
38
+ diagnostic = Diagnostic.from(error, backup: outcome.backup, phase: :tftp_backup)
39
+ outcome.with(status: :reported_with_error, error_code: diagnostic.error_code,
40
+ error_type: diagnostic.error_type, diagnostic: diagnostic)
25
41
  end
26
42
 
27
- def verify_outcome(outcome, started_at: outcome.started_at, previous: nil)
43
+ # 调用者必须在目标锁内先捕获基线;nil 明确表示上传前文件缺失,不能事后猜测。
44
+ def verify_outcome(outcome, started_at:, previous:, expected_path:)
28
45
  return outcome unless @root && started_at && outcome.backup.is_a?(TftpReceipt) && outcome.backup.path
46
+ return outcome unless outcome.backup.path == expected_path
29
47
 
30
48
  verify(outcome, outcome.backup, started_at: started_at, previous: previous)
31
49
  end
32
50
 
33
51
  private
34
52
 
35
- def verify(outcome, receipt, started_at: outcome.started_at, previous: nil)
36
- path = File.join(@root, receipt.path)
53
+ def server_path(relative_path)
54
+ path = File.join(@root, relative_path)
37
55
  parent = File.realpath(File.dirname(path))
38
- return outcome unless parent == @root || parent.start_with?(@root + File::SEPARATOR)
56
+ unless parent == @root || parent.start_with?(@root + File::SEPARATOR)
57
+ raise ArgumentError, "TFTP verification path escaped its root"
58
+ end
59
+
60
+ path
61
+ end
39
62
 
63
+ def verify(outcome, receipt, started_at:, previous:)
64
+ path = server_path(receipt.path)
40
65
  size = nil
41
66
  fingerprint = Storage::SafeFile.open(path, missing: true) do |file, stat|
42
67
  next unless stat.size.positive? && stat.mtime >= started_at
@@ -1,6 +1,6 @@
1
1
  # frozen_string_literal: true
2
2
 
3
- require_relative "settings"
3
+ require_relative "../execution_pool"
4
4
 
5
5
  module Net
6
6
  module Connector
@@ -9,11 +9,7 @@ module Net
9
9
  class Worker
10
10
  # 内部预检与实际执行共用规则,不为校验创建工作线程池对象。
11
11
  def self.validate_concurrency!(concurrency)
12
- unless concurrency.is_a?(Integer) && (1..Settings::MAX_CONCURRENCY).cover?(concurrency)
13
- raise ArgumentError, "concurrency must be an Integer in 1..#{Settings::MAX_CONCURRENCY}"
14
- end
15
-
16
- concurrency
12
+ ExecutionPool.validate_concurrency!(concurrency)
17
13
  end
18
14
 
19
15
  # 校验并保存最大设备并发数。
@@ -23,47 +19,12 @@ module Net
23
19
 
24
20
  # 按清单顺序收集结果,并隔离设备及回调异常。
25
21
  def run(tasks, outcomes:, on_error:, on_start: nil, on_result: nil)
26
- queue = Queue.new
27
- tasks.each { |task| queue << task }
28
- worker_count = [tasks.size, @concurrency].min
29
- worker_count.times { queue << nil }
30
22
  callback_errors = Queue.new
31
- finished = Queue.new
32
- threads = []
33
- completed = false
34
- begin
35
- # 创建线程也属于批次生命周期;后续创建失败时要关闭已有任务。
36
- worker_count.times do
37
- threads << Thread.new do
38
- Thread.current.report_on_exception = false
39
- while (task = queue.pop)
40
- index, device = task
41
- result = run_task(device, on_error, on_start, callback_errors) { yield device }
42
- outcomes[index] = result
43
- Array(on_result).each do |callback|
44
- notify(callback, result, host: device.host, errors: callback_errors)
45
- end
46
- end
47
- ensure
48
- finished << Thread.current
49
- end
50
- end
51
- # 按完成顺序观察失败,避免后启动线程的中断被前面的慢设备阻塞。
52
- # 结果仍写回清单槽位,完成通知不会改变公开结果的顺序。
53
- worker_count.times { finished.pop.value }
54
- completed = true
55
- ensure
56
- # 中断或某个 worker 抛出非 StandardError 时,不让其他设备任务
57
- # 在调用方已经退出后继续运行;Thread#kill 会执行任务自身的 ensure。
58
- unless completed
59
- threads.each(&:kill)
60
- threads.each do |thread|
61
- begin
62
- thread.join
63
- rescue Exception # rubocop:disable Lint/RescueException -- 等待全部工作线程停止,不让收尾异常覆盖原中断。
64
- nil
65
- end
66
- end
23
+ ExecutionPool.new(concurrency: @concurrency).map(tasks) do |index, device|
24
+ result = run_task(device, on_error, on_start, callback_errors) { yield device }
25
+ outcomes[index] = result
26
+ Array(on_result).each do |callback|
27
+ notify(callback, result, host: device.host, errors: callback_errors)
67
28
  end
68
29
  end
69
30
  Array.new(callback_errors.size) { callback_errors.pop }.freeze
@@ -43,7 +43,7 @@ module Net
43
43
  lines = neighbor_lines(output)
44
44
  return unless lines.all? do |line|
45
45
  interface = line.split.fetch(local_column, nil)
46
- interface&.match?(/\A[A-Za-z][A-Za-z-]*\d+(?:\/\d+)+\z/)
46
+ interface&.match?(/\A(?:[A-Za-z][A-Za-z-]*\d+(?:\/\d+)+|(?:Bridge-Aggregation|BAGG|Route-Aggregation|RAGG)\d+)(?:[:.]\d+)*\z/i)
47
47
  end
48
48
 
49
49
  lines.size