net-connector 0.4.0 → 0.4.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (68) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +110 -68
  3. data/CONTRIBUTING.md +40 -0
  4. data/README.md +140 -95
  5. data/SECURITY.md +11 -0
  6. data/docs/VERIFICATION.md +129 -5
  7. data/docs/architecture.md +321 -248
  8. data/lib/net/connector/device/base.rb +12 -4
  9. data/lib/net/connector/device/interface_description.rb +3 -0
  10. data/lib/net/connector/device/profile/builder.rb +236 -0
  11. data/lib/net/connector/device/profile.rb +2 -227
  12. data/lib/net/connector/device/running_config/strategy.rb +5 -0
  13. data/lib/net/connector/device/running_config.rb +7 -0
  14. data/lib/net/connector/engine/command.rb +14 -2
  15. data/lib/net/connector/engine/configuration.rb +10 -7
  16. data/lib/net/connector/engine/dialogue.rb +53 -28
  17. data/lib/net/connector/engine/errors.rb +41 -54
  18. data/lib/net/connector/engine/execution.rb +32 -5
  19. data/lib/net/connector/engine/log.rb +23 -17
  20. data/lib/net/connector/engine/session.rb +39 -7
  21. data/lib/net/connector/engine/terminal_renderer.rb +7 -4
  22. data/lib/net/connector/netdisco/batch.rb +31 -3
  23. data/lib/net/connector/netdisco/cli.rb +92 -50
  24. data/lib/net/connector/netdisco/client.rb +206 -73
  25. data/lib/net/connector/netdisco/config_file.rb +26 -4
  26. data/lib/net/connector/netdisco/device.rb +4 -8
  27. data/lib/net/connector/netdisco/diagnostic.rb +91 -0
  28. data/lib/net/connector/netdisco/fleet.rb +105 -108
  29. data/lib/net/connector/netdisco/inventory_budget.rb +49 -0
  30. data/lib/net/connector/netdisco/plan.rb +77 -0
  31. data/lib/net/connector/netdisco/planner.rb +12 -23
  32. data/lib/net/connector/netdisco/report.rb +94 -0
  33. data/lib/net/connector/netdisco/rules.rb +28 -5
  34. data/lib/net/connector/netdisco/settings.rb +194 -85
  35. data/lib/net/connector/netdisco/worker.rb +43 -27
  36. data/lib/net/connector/operations/backup_lock.rb +116 -0
  37. data/lib/net/connector/operations/local_backup.rb +49 -9
  38. data/lib/net/connector/operations/parse_output.rb +20 -3
  39. data/lib/net/connector/operations/private_file.rb +94 -5
  40. data/lib/net/connector/operations/safe_file.rb +62 -0
  41. data/lib/net/connector/operations/saved_config/legacy_index.rb +109 -0
  42. data/lib/net/connector/operations/saved_config.rb +40 -10
  43. data/lib/net/connector/operations/tftp/file_upload.rb +55 -0
  44. data/lib/net/connector/operations/tftp/strategy.rb +9 -11
  45. data/lib/net/connector/operations/tftp_backup.rb +78 -15
  46. data/lib/net/connector/operations/tftp_receipt.rb +73 -0
  47. data/lib/net/connector/operations/topology/immediate_strategy.rb +45 -0
  48. data/lib/net/connector/operations/topology/strategy.rb +34 -0
  49. data/lib/net/connector/operations/topology.rb +98 -29
  50. data/lib/net/connector/operations.rb +6 -0
  51. data/lib/net/connector/vendor/cisco_ios/running_config.rb +1 -0
  52. data/lib/net/connector/vendor/cisco_ios/tftp_backup.rb +9 -2
  53. data/lib/net/connector/vendor/cisco_ios/topology.rb +18 -3
  54. data/lib/net/connector/vendor/cisco_nxos/running_config.rb +1 -0
  55. data/lib/net/connector/vendor/cisco_nxos/tftp_backup.rb +8 -1
  56. data/lib/net/connector/vendor/h3c/tftp_backup.rb +7 -18
  57. data/lib/net/connector/vendor/h3c/topology.rb +17 -3
  58. data/lib/net/connector/vendor/hillstone/running_config.rb +1 -0
  59. data/lib/net/connector/vendor/hillstone/tftp_backup.rb +16 -3
  60. data/lib/net/connector/vendor/hillstone/topology.rb +17 -3
  61. data/lib/net/connector/vendor/huawei/tftp_backup.rb +8 -18
  62. data/lib/net/connector/vendor/palo_alto/running_config.rb +4 -0
  63. data/lib/net/connector/vendor/palo_alto/tftp_backup.rb +19 -8
  64. data/lib/net/connector/vendor/palo_alto/topology.rb +16 -1
  65. data/lib/net/connector/vendor/radware/tftp_backup.rb +11 -4
  66. data/lib/net/connector/vendor/radware/topology.rb +3 -0
  67. data/lib/net/connector/version.rb +1 -1
  68. metadata +60 -28
@@ -10,7 +10,13 @@ module Net
10
10
  class ConfigFile
11
11
  FIELDS = {
12
12
  "netdisco" => { "url" => ["NETDISCO_URL", :string],
13
- "page_size" => ["NETDISCO_PAGE_SIZE", :integer] },
13
+ "page_size" => ["NETDISCO_PAGE_SIZE", :integer],
14
+ "max_pages" => ["NETDISCO_MAX_PAGES", :integer],
15
+ "max_response_bytes" => ["NETDISCO_MAX_RESPONSE_BYTES", :integer],
16
+ "max_inventory_bytes" => ["NETDISCO_MAX_INVENTORY_BYTES", :integer],
17
+ "max_devices" => ["NETDISCO_MAX_DEVICES", :integer],
18
+ "inventory_timeout" => ["NETDISCO_INVENTORY_TIMEOUT", :number],
19
+ "allow_insecure_http" => ["NETDISCO_ALLOW_INSECURE_HTTP", :boolean] },
14
20
  "backup" => { "directory" => ["NET_CONNECTOR_BACKUP_DIRECTORY", :string],
15
21
  "concurrency" => ["NET_CONNECTOR_CONCURRENCY", :integer],
16
22
  "limit_per_vendor" => ["NET_CONNECTOR_SAMPLE_PER_VENDOR", :integer] },
@@ -21,12 +27,16 @@ module Net
21
27
  "host_overrides" => ["NET_CONNECTOR_HOST_OVERRIDES", :json],
22
28
  "device_rules" => ["NET_CONNECTOR_DEVICE_RULES", :json] },
23
29
  "ssh" => { "protocol" => ["NET_CONNECTOR_PROTOCOL", :string],
30
+ "max_script_output_bytes" => ["NET_CONNECTOR_MAX_SCRIPT_OUTPUT_BYTES", :optional_integer],
24
31
  "known_hosts" => ["NET_CONNECTOR_KNOWN_HOSTS", :string],
25
32
  "host_key_policy" => ["NET_CONNECTOR_HOST_KEY_POLICY", :string],
26
33
  "log_directory" => ["NET_CONNECTOR_LOG_DIRECTORY", :string],
27
34
  "log_level" => ["NET_CONNECTOR_LOG_LEVEL", :string] },
28
35
  "tftp" => { "server" => ["TFTP_HOST", :string],
29
- "vrfs" => ["NET_CONNECTOR_TFTP_VRFS", :json] }
36
+ "vrfs" => ["NET_CONNECTOR_TFTP_VRFS", :json],
37
+ "h3c_source_file" => ["NET_CONNECTOR_H3C_TFTP_SOURCE_FILE", :string],
38
+ "h3c_wireless_source_file" => ["NET_CONNECTOR_H3C_WIRELESS_TFTP_SOURCE_FILE", :string],
39
+ "huawei_source_file" => ["NET_CONNECTOR_HUAWEI_TFTP_SOURCE_FILE", :string] }
30
40
  }.freeze
31
41
 
32
42
  # 安全读取 YAML 配置并映射到环境变量格式。
@@ -52,7 +62,7 @@ module Net
52
62
  end
53
63
  end
54
64
  rescue Psych::Exception => error
55
- raise ArgumentError, "invalid YAML config (#{error.class})"
65
+ raise ArgumentError, "invalid YAML config (#{error.class})", cause: nil
56
66
  end
57
67
 
58
68
  # 按字段类型校验并编码单项配置。
@@ -62,9 +72,21 @@ module Net
62
72
  raise ArgumentError, "#{section}.#{name} must be a nonempty String" unless value.is_a?(String) && !value.strip.empty?
63
73
 
64
74
  value
65
- when :integer
75
+ when :integer, :optional_integer
76
+ return nil if type == :optional_integer && value.nil?
77
+
66
78
  raise ArgumentError, "#{section}.#{name} must be a positive Integer" unless value.is_a?(Integer) && value.positive?
67
79
 
80
+ value.to_s
81
+ when :number
82
+ unless value.is_a?(Numeric) && value.real? && value.finite? && value.positive? && value.to_f.finite?
83
+ raise ArgumentError, "#{section}.#{name} must be a positive finite number"
84
+ end
85
+
86
+ value.to_s
87
+ when :boolean
88
+ raise ArgumentError, "#{section}.#{name} must be true or false" unless [true, false].include?(value)
89
+
68
90
  value.to_s
69
91
  when :list
70
92
  unless value.is_a?(Array) && value.all? { |item| item.is_a?(String) && !item.empty? && !item.include?(",") }
@@ -48,18 +48,14 @@ module Net
48
48
 
49
49
  # 生成设备 CLI 可用的 ASCII 远端文件名。
50
50
  def tftp_filename
51
- return "running-config.xml" if vendor == :palo_alto
52
-
53
51
  label = name.to_s.encode(Encoding::US_ASCII, invalid: :replace, undef: :replace, replace: "-")
54
52
  .gsub(/[^A-Za-z0-9._-]+/, "-").byteslice(0, 180)
55
53
  .gsub(/\A[._-]+|[._-]+\z/, "")
56
54
  label = vendor&.to_s || "device" if label.empty?
57
- extension = case vendor
58
- when :radware then "tgz"
59
- when :hillstone then "dat"
60
- else "cfg"
61
- end
62
- TftpTarget.filename(host, extension: extension, label: label)
55
+ strategy = Net::Connector.vendor_class(vendor).profile.tftp_strategy if vendor
56
+ # 旧版自定义策略只需实现实例接口,缺少命名接口时沿用通用 cfg 名称。
57
+ strategy = Operations::Tftp::Strategy unless strategy.respond_to?(:filename)
58
+ TftpTarget.validate_path!(strategy.filename(host, label: label))
63
59
  end
64
60
 
65
61
  # 按厂商及设备地址实例化连接器。
@@ -0,0 +1,91 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Net
4
+ module Connector
5
+ module Netdisco
6
+ # 报告是新的诊断通道:只接受固定词表,不展开异常消息、调用栈、命令、source 或 line。
7
+ Diagnostic = Data.define(:error_code, :error_type, :phase, :underlying_type,
8
+ :artifact_state, :artifact_phase, :verification)
9
+
10
+ class Diagnostic
11
+ CONNECTOR_TYPES = %w[Error ConnectionError AuthenticationError LoginTimeout CommandTimeout WriteTimeout
12
+ PromptError ConnectionClosed TransportError DeviceError ScriptError InternalError
13
+ OutputLimitExceeded ScriptOutputLimitExceeded SessionBusy UnsupportedOperation LogError ParsingError BackupBusy
14
+ BackupPersistenceError TftpCompletionError SavedConfigChanged].freeze
15
+ TYPES = (CONNECTOR_TYPES.map { |name| "Net::Connector::#{name}" } +
16
+ %w[StandardError RuntimeError ArgumentError TypeError IOError EOFError SystemCallError ThreadError
17
+ RangeError KeyError IndexError NoMethodError NameError FrozenError NotImplementedError
18
+ Timeout::Error IO::TimeoutError Errno::EIO Errno::EACCES Errno::EPERM Errno::ENOSPC Errno::EROFS
19
+ Errno::ENOENT Errno::ENOTDIR Errno::EISDIR Errno::ELOOP Errno::EINVAL Errno::ENOSYS
20
+ Errno::ENOTSUP Errno::EOPNOTSUPP Errno::ETIMEDOUT Errno::ECONNREFUSED Errno::ECONNRESET
21
+ Net::Connector::Operations::PrivateFile::WriteError
22
+ Net::Connector::Operations::PrivateFile::PersistenceError
23
+ Net::Connector::Operations::PrivateFile::DirectorySyncUnsupported]).freeze
24
+ CODES = (CONNECTOR_TYPES.map { |name| name.gsub(/([a-z])([A-Z])/, '\1_\2').downcase.to_sym } +
25
+ %i[authentication_failed connection_failed connection_refused no_route connection_reset
26
+ connection_timeout host_key_changed host_key_untrusted rsa_too_small unsupported_cipher
27
+ ambiguous_description ambiguous_neighbor confirmation_required description_stages_unsupported
28
+ description_unconfirmed description_unsupported incomplete_configuration interface_missing
29
+ neighbor_discovery_unsupported parse_failed invalid_output_encoding persistence_unconfirmed stale_plan startup_config_missing
30
+ template_invalid template_missing template_unreadable transfer_failed transfer_finalize_failed
31
+ transfer_path_mismatch transfer_path_unconfirmed transfer_unconfirmed uncommitted_configuration
32
+ unrecognized_output unsupported_configuration_format verification_plan_changed candidate_isolation_unavailable
33
+ backup_durability_unsupported backup_finalize_failed backup_persistence_unconfirmed
34
+ file_finalize_failed file_persistence_unconfirmed file_write_failed directory_sync_unsupported]).freeze
35
+ PHASES = %i[connect login enable command script read write close logging backup collect discover parse
36
+ plan apply verify persist tftp_backup report callback].freeze
37
+ ARTIFACT_STATES = %i[not_committed committed durable reported_uploaded].freeze
38
+ ARTIFACT_PHASES = %i[resolve_parent temporary_file write file_sync rename directory_open directory_sync
39
+ cleanup complete path finalize].freeze
40
+ private_constant :CONNECTOR_TYPES, :TYPES, :CODES, :PHASES, :ARTIFACT_STATES, :ARTIFACT_PHASES
41
+
42
+ # 包括手工构造的结果在内,v2 都不能携带自由文本诊断值。
43
+ def initialize(error_code: nil, error_type: nil, phase: nil, underlying_type: nil,
44
+ artifact_state: nil, artifact_phase: nil, verification: nil)
45
+ super(error_code: CODES.include?(error_code) ? error_code : nil,
46
+ error_type: self.class.type(error_type), phase: PHASES.include?(phase) ? phase : nil,
47
+ underlying_type: self.class.type(underlying_type),
48
+ artifact_state: ARTIFACT_STATES.include?(artifact_state) ? artifact_state : nil,
49
+ artifact_phase: ARTIFACT_PHASES.include?(artifact_phase) ? artifact_phase : nil,
50
+ verification: verification == :device_reported ? verification : nil)
51
+ end
52
+
53
+ def self.type(name)
54
+ return if name.nil?
55
+
56
+ name.instance_of?(String) && TYPES.include?(name) ? name.dup.freeze : "StandardError"
57
+ end
58
+
59
+ # 读取已知错误的白名单属性后立即丢弃异常引用,不把回调对象保留进批次。
60
+ def self.from(error, backup: nil, phase: nil)
61
+ return unless error
62
+
63
+ fields = { error_type: type(error.class.name) || "StandardError", phase: phase }
64
+ if error.is_a?(Net::Connector::Error)
65
+ fields.merge!(error_code: error.code, phase: phase || error.phase)
66
+ fields[:underlying_type] = error.underlying.type if error.underlying.instance_of?(UnderlyingError)
67
+ end
68
+ new(**fields.merge(completion_fields(error, backup, phase)))
69
+ rescue StandardError
70
+ new(error_type: "StandardError", phase: phase)
71
+ end
72
+
73
+ def self.completion_fields(error, backup, phase)
74
+ if error.instance_of?(BackupPersistenceError) && backup.instance_of?(Backup) && error.backup.equal?(backup)
75
+ { artifact_state: error.receipt.state, artifact_phase: error.receipt.phase,
76
+ underlying_type: error.underlying_type }
77
+ elsif error.instance_of?(TftpCompletionError) && backup.instance_of?(TftpBackup) && error.transfer.equal?(backup)
78
+ { artifact_state: :reported_uploaded, artifact_phase: error.code == :transfer_finalize_failed ? :finalize : :path,
79
+ verification: :device_reported, underlying_type: error.underlying_type }
80
+ elsif phase == :report && Operations::PrivateFile.receipt_error?(error)
81
+ { error_code: error.code, artifact_state: error.receipt.state, artifact_phase: error.receipt.phase,
82
+ underlying_type: error.underlying_type }
83
+ else
84
+ {}
85
+ end
86
+ end
87
+ private_class_method :completion_fields
88
+ end
89
+ end
90
+ end
91
+ end
@@ -8,90 +8,117 @@ module Net
8
8
  module Netdisco
9
9
  # 基于已验证的清单快照组织有上限的并发备份。
10
10
  class Fleet
11
+ DEFAULT_SETTING = Object.new.freeze
12
+ private_constant :DEFAULT_SETTING
13
+
11
14
  # 组装清单客户端、规则、凭据与结果存储。
12
15
  def initialize(client: nil, settings: Settings.new, rules: nil, credentials: nil, connector_factory: nil,
13
16
  result_store: ResultStore::Text.new)
14
17
  @settings = settings
15
- @client = client || settings.client
16
- @rules = rules || settings.rules
17
- @credentials = credentials || settings.method(:credentials_for)
18
+ @client = client
19
+ @rules = rules
20
+ @credentials = credentials
18
21
  @connector_factory = connector_factory || ->(device, connection_settings) { device.connector(**connection_settings) }
19
22
  @result_store = result_store
20
- raise ArgumentError, "credentials must respond to call" unless @credentials.respond_to?(:call)
23
+ raise ArgumentError, "credentials must respond to call" if @credentials && !@credentials.respond_to?(:call)
21
24
  raise ArgumentError, "result_store must respond to write" if result_store && !result_store.respond_to?(:write)
22
25
  end
23
26
 
24
27
  # 拉取并标记重复管理地址的设备清单。
25
28
  def devices
26
- rows = @client.devices
27
- raise Client::Error, "Netdisco returned an invalid device inventory" unless rows.is_a?(Array)
28
-
29
- devices = rows.map { |row| Device.from_row(row, rules: @rules) }
30
- duplicates = devices.group_by(&:host).reject { |host, entries| host.nil? || entries.size == 1 }
31
- devices.map do |device|
32
- duplicates.key?(device.host) ? device.with(issue: :duplicate_host) : device
33
- end.freeze
29
+ inventory(@settings.snapshot(mode: :inventory))
34
30
  end
35
31
 
36
32
  # 规划本地配置备份的设备集合。
37
33
  def plan_backup(limit_per_vendor: nil)
38
- Planner.new(devices).call(mode: :backup, limit_per_vendor: limit_per_vendor)
34
+ build_plan(:backup, limit_per_vendor, @settings.snapshot(mode: :backup))
39
35
  end
40
36
 
41
37
  # 规划 TFTP 配置备份的设备集合。
42
38
  def plan_tftp_backup(limit_per_vendor: 5)
43
- Planner.new(devices).call(mode: :tftp, limit_per_vendor: limit_per_vendor)
39
+ build_plan(:tftp, limit_per_vendor, @settings.snapshot(mode: :tftp))
44
40
  end
45
41
 
46
42
  # 先拉取完整清单,再并发采集并保存设备配置。
47
- def backup_all(directory: @settings.backup_directory, concurrency: @settings.concurrency,
48
- limit_per_vendor: nil, plan: nil, on_start: nil, on_result: nil, on_change: nil)
43
+ def backup_all(directory: DEFAULT_SETTING, concurrency: DEFAULT_SETTING,
44
+ limit_per_vendor: nil, plan: nil, on_start: nil, on_result: nil, on_change: nil,
45
+ success_policy: :strict, report_schema: nil)
46
+ reporting = Report.options(policy: success_policy, schema: report_schema)
47
+ policy = @settings.snapshot(mode: :backup)
48
+ directory = policy.backup_directory if directory.equal?(DEFAULT_SETTING)
49
+ concurrency = policy.concurrency if concurrency.equal?(DEFAULT_SETTING)
49
50
  raise ArgumentError, "directory must be a nonempty String" unless directory.is_a?(String) && !directory.empty?
51
+ Worker.new(concurrency: concurrency)
50
52
  Planner.validate_limit!(limit_per_vendor)
51
53
  validate_callback!(on_result, :on_result)
52
54
  validate_callback!(on_start, :on_start)
53
55
  validate_callback!(on_change, :on_change)
54
56
 
55
- plan ||= plan_backup(limit_per_vendor: limit_per_vendor)
57
+ plan ||= build_plan(:backup, limit_per_vendor, policy)
56
58
  validate_plan!(plan, :backup)
57
59
  callbacks = [on_result]
58
60
  callbacks << ->(item) { on_change.call(item) if item.backup.is_a?(Backup) && item.backup.changed? } if on_change
59
61
  target_directory = File.expand_path(directory)
62
+ saved_config = Operations::SavedConfig.new(directory: target_directory, indexed: true)
60
63
  run_batch(:backup, plan, concurrency: concurrency, report_directory: directory,
61
- output_directory: target_directory, on_start: on_start, on_result: callbacks.compact) do |device, log_directory|
62
- backup_one(device, target_directory, log_directory)
64
+ output_directory: target_directory, policy: policy, reporting: reporting,
65
+ on_start: on_start, on_result: callbacks.compact) do |device, log_directory|
66
+ backup_one(device, target_directory, log_directory, policy, saved_config)
63
67
  end
64
68
  end
65
69
 
66
- # 让设备主动导出配置,避免 PAN-OS 固定远端文件名在同批任务中冲突。
67
- def tftp_backup_all(server:, source_files: {}, concurrency: @settings.concurrency,
70
+ # 让设备主动导出配置,在执行前检查同批目标文件名冲突。
71
+ def tftp_backup_all(server:, source_files: {}, concurrency: DEFAULT_SETTING,
68
72
  limit_per_vendor: 5, vrfs: {}, on_start: nil, on_result: nil, plan: nil,
69
- report_directory: @settings.backup_directory)
70
- TftpTarget.new(host: server, path: "preflight.cfg")
73
+ report_directory: DEFAULT_SETTING, success_policy: :strict, report_schema: nil)
74
+ reporting = Report.options(policy: success_policy, schema: report_schema)
75
+ policy = @settings.snapshot(mode: :tftp)
76
+ concurrency = policy.concurrency if concurrency.equal?(DEFAULT_SETTING)
77
+ report_directory = policy.backup_directory if report_directory.equal?(DEFAULT_SETTING)
78
+ Worker.new(concurrency: concurrency)
79
+ server = TftpTarget.new(host: server, path: "preflight.cfg").host
71
80
  raise ArgumentError, "source_files must be a Hash" unless source_files.is_a?(Hash)
72
81
  source_files.each_value { |source_file| TftpTarget.validate_source_file!(source_file) }
73
- validate_vrfs!(vrfs)
82
+ source_files = source_files.transform_values { |value| value.dup.freeze }.freeze
83
+ vrfs = Settings.validate_vrfs!(vrfs).transform_values { |value| value.dup.freeze }.freeze
74
84
  Planner.validate_limit!(limit_per_vendor)
75
85
  validate_callback!(on_result, :on_result)
76
86
  validate_callback!(on_start, :on_start)
77
87
 
78
- plan ||= plan_tftp_backup(limit_per_vendor: limit_per_vendor)
88
+ plan ||= build_plan(:tftp, limit_per_vendor, policy)
79
89
  validate_plan!(plan, :tftp)
80
90
  run_batch(:tftp, plan, concurrency: concurrency, report_directory: report_directory,
81
- on_start: on_start, on_result: on_result) do |device, log_directory|
82
- tftp_backup_one(device, server, source_files, log_directory, vrfs)
91
+ policy: policy, reporting: reporting, on_start: on_start, on_result: on_result) do |device, log_directory|
92
+ tftp_backup_one(device, server, source_files, log_directory, vrfs, policy)
83
93
  end
84
94
  end
85
95
 
86
96
  private
87
97
 
98
+ # 策略在清单请求之前固定;注入的清单客户端仍由调用方负责其自身生命周期。
99
+ def inventory(policy)
100
+ rows = (@client || @settings.client(policy: policy)).devices
101
+ raise Client::Error, "Netdisco returned an invalid device inventory" unless rows.is_a?(Array)
102
+
103
+ rules = @rules || policy.rules
104
+ devices = rows.map { |row| Device.from_row(row, rules: rules) }
105
+ duplicates = devices.group_by(&:host).reject { |host, entries| host.nil? || entries.size == 1 }
106
+ devices.map { |device| duplicates.key?(device.host) ? device.with(issue: :duplicate_host) : device }.freeze
107
+ end
108
+
109
+ def build_plan(mode, limit, policy)
110
+ Planner.validate_limit!(limit)
111
+ Planner.new(inventory(policy)).call(mode: mode, limit_per_vendor: limit)
112
+ end
113
+
88
114
  # 统一组织设备任务、日志目录与报告;Worker 保证单台设备失败不影响其他设备。
89
- def run_batch(mode, plan, concurrency:, report_directory:, on_start:, on_result:, output_directory: nil)
115
+ def run_batch(mode, plan, concurrency:, report_directory:, policy:, reporting:, on_start:, on_result:, output_directory: nil)
90
116
  worker = Worker.new(concurrency: concurrency)
91
117
  started_at = Time.now.utc
118
+ started = monotonic
92
119
  outcomes = plan.outcomes.dup
93
120
  FileUtils.mkdir_p(output_directory, mode: 0o700) if output_directory && !plan.ready.empty?
94
- log_directory = @settings.log_directory
121
+ log_directory = policy.log_directory
95
122
  FileUtils.mkdir_p(log_directory, mode: 0o700) if log_directory && !plan.ready.empty?
96
123
  callback_errors = worker.run(plan.ready, outcomes: outcomes, on_start: on_start, on_result: on_result,
97
124
  on_error: ->(device, error) { outcome(device, :failed, error: error) }) do |device|
@@ -100,9 +127,14 @@ module Net
100
127
  batch = Batch.new(mode: mode, outcomes: outcomes.freeze, started_at: started_at,
101
128
  finished_at: Time.now.utc, callback_errors: callback_errors,
102
129
  report_location: nil, report_error: nil)
130
+ if reporting.fetch(:schema) == 2
131
+ batch = Report.new(batch, policy: reporting.fetch(:policy), duration_ms: ((monotonic - started) * 1000).round)
132
+ end
103
133
  save_report(batch, directory: report_directory)
104
134
  end
105
135
 
136
+ def monotonic = Process.clock_gettime(Process::CLOCK_MONOTONIC)
137
+
106
138
  # 校验批量任务回调接口。
107
139
  def validate_callback!(callback, name)
108
140
  raise ArgumentError, "#{name} must respond to call" if callback && !callback.respond_to?(:call)
@@ -113,83 +145,35 @@ module Net
113
145
  return batch unless @result_store
114
146
 
115
147
  batch.with(report_location: @result_store.write(batch, directory: directory))
148
+ rescue Operations::PrivateFile::WriteError => error
149
+ # 报告也可能已经原子替换;保留已知位置,不将持久性失败误写成完全没有产物。
150
+ location = error.receipt.path if Operations::PrivateFile.receipt_error?(error) && error.receipt.committed?
151
+ return batch.with_report_error(error, location: location) if batch.instance_of?(Report)
152
+
153
+ batch.with(report_location: location, report_error: error.class.name)
116
154
  rescue StandardError => error
117
- batch.with(report_error: error.class.name)
118
- end
155
+ return batch.with_report_error(error) if batch.instance_of?(Report)
119
156
 
120
- # 校验厂商 VRF 映射和名称。
121
- def validate_vrfs!(vrfs)
122
- unless vrfs.is_a?(Hash) && vrfs.all? { |vendor, value|
123
- %i[cisco_nxos hillstone].include?(vendor) && value.is_a?(String) &&
124
- value.match?(/\A[A-Za-z0-9_][A-Za-z0-9_.-]*\z/)
125
- }
126
- raise ArgumentError, "vrfs must map supported vendor names to safe VRF names"
127
- end
157
+ batch.with(report_error: error.class.name)
128
158
  end
129
159
 
130
160
  # 确认传入计划属于当前备份模式。
131
161
  def validate_plan!(plan, mode)
132
162
  raise ArgumentError, "plan must be a #{mode} Netdisco plan" unless plan.is_a?(Plan) && plan.mode == mode
133
163
 
134
- inventory = plan.inventory
135
- ready = plan.ready
136
- outcomes = plan.outcomes
137
- unless inventory.is_a?(Array) && inventory.all?(Device) && ready.is_a?(Array) &&
138
- outcomes.is_a?(Array) && outcomes.size == inventory.size
139
- raise ArgumentError, "plan inventory and outcomes must have the same device slots"
140
- end
141
-
142
- selected = {}
143
- previous_index = -1
144
- ready.each do |task|
145
- unless task.is_a?(Array) && task.size == 2 && task.first.is_a?(Integer)
146
- raise ArgumentError, "plan contains an invalid device task"
147
- end
148
-
149
- index, device = task
150
- unless index > previous_index && index < inventory.size && inventory[index].equal?(device) && device.ready?
151
- raise ArgumentError, "plan tasks must reference ready inventory devices in order"
152
- end
153
-
154
- selected[index] = true
155
- previous_index = index
156
- end
157
- if mode == :tftp && ready.count { |_index, device| device.vendor == :palo_alto } > 1
158
- raise ArgumentError, "plan cannot upload multiple Palo Alto devices to one TFTP filename"
159
- end
160
-
161
- inventory.each_with_index do |device, index|
162
- result = outcomes[index]
163
- next if selected[index] && result.nil?
164
-
165
- unless !selected[index] && valid_skipped_outcome?(result, device, mode)
166
- raise ArgumentError, "plan outcome does not match its inventory slot"
167
- end
168
- end
169
- end
170
-
171
- # 未执行设备必须保留与清单一致的跳过原因,且不能带有执行产物。
172
- def valid_skipped_outcome?(result, device, mode)
173
- return false unless result.is_a?(Outcome) && result.device.equal?(device) && result.backup.nil? &&
174
- result.error_code.nil? && result.error_type.nil? &&
175
- result.started_at.nil? && result.finished_at.nil?
176
-
177
- return result.status == device.issue unless device.ready?
178
-
179
- result.status == :sample_limit ||
180
- (mode == :tftp && device.vendor == :palo_alto && result.status == :remote_filename_collision)
164
+ plan.validate!
181
165
  end
182
166
 
183
167
  # 构造包含错误类型的单台设备结果。
184
168
  def outcome(device, status, backup: nil, error: nil)
185
169
  Outcome.new(device: device, status: status, backup: backup,
186
170
  error_code: error.respond_to?(:code) ? error.code : nil,
187
- error_type: error&.class&.name)
171
+ error_type: error&.class&.name, diagnostic: Diagnostic.from(error, backup: backup))
188
172
  end
189
173
 
190
174
  # 执行单台设备的 TFTP 导出。
191
- def tftp_backup_one(device, server, source_files, log_directory, vrfs)
192
- run_one(device, log_directory, success_status: :reported_uploaded,
175
+ def tftp_backup_one(device, server, source_files, log_directory, vrfs, policy)
176
+ run_one(device, log_directory, policy, success_status: :reported_uploaded,
193
177
  close_error_status: :reported_with_error, backup_class: TftpBackup) do |connector|
194
178
  source_file = source_files.fetch(device.vendor, nil)
195
179
  transfer_settings = { host: server, path: device.tftp_filename, source_file: source_file }
@@ -199,25 +183,26 @@ module Net
199
183
  end
200
184
 
201
185
  # 执行单台设备的本地配置采集。
202
- def backup_one(device, directory, log_directory)
186
+ def backup_one(device, directory, log_directory, policy, saved_config)
203
187
  destination = File.join(directory, device.backup_filename)
204
- previous = Operations::SavedConfig.new(directory: directory).find(device.host, required: false)
205
- previous_digest = Digest::SHA256.file(previous).hexdigest if previous && previous != destination
206
- run_one(device, log_directory, success_status: :backed_up,
207
- close_error_status: :saved_with_error, backup_class: Backup) do |connector|
208
- backup = connector.backup(path: destination)
209
- if backup.is_a?(Backup) && backup.change == :created && previous_digest
210
- backup.with(previous_sha256: previous_digest,
211
- change: backup.sha256 == previous_digest ? :unchanged : :changed)
212
- else
213
- backup
188
+ Operations::BackupLock.synchronize(destination, host: device.host) do |path_lock|
189
+ previous = saved_config.fingerprint(device.host, required: false)
190
+ previous_digest = previous.sha256 if previous && previous.path != destination
191
+ result = run_one(device, log_directory, policy, success_status: :backed_up,
192
+ close_error_status: :saved_with_error, backup_class: Backup) do |connector|
193
+ path_lock.delegate { connector.backup(path: destination) }
214
194
  end
195
+ backup = result.backup
196
+ next result unless backup.is_a?(Backup) && backup.change == :created && previous_digest
197
+
198
+ result.with(backup: backup.with(previous_sha256: previous_digest,
199
+ change: backup.sha256 == previous_digest ? :unchanged : :changed))
215
200
  end
216
201
  end
217
202
 
218
203
  # 隔离单台设备的凭据、连接、操作和关闭异常。
219
- def run_one(device, log_directory, success_status:, close_error_status:, backup_class:)
220
- connection_settings = @credentials.call(device)
204
+ def run_one(device, log_directory, policy, success_status:, close_error_status:, backup_class:)
205
+ connection_settings = @credentials ? @credentials.call(device) : @settings.device_credentials_for(device)
221
206
  return outcome(device, :missing_credentials) if connection_settings.nil?
222
207
  unless connection_settings.is_a?(Hash) && connection_settings.keys.all?(Symbol) &&
223
208
  connection_settings[:username].is_a?(String) && !connection_settings[:username].empty? &&
@@ -225,20 +210,36 @@ module Net
225
210
  raise ArgumentError, "credential resolver must return settings with username and without host"
226
211
  end
227
212
 
213
+ connection_settings = policy.connection_options(device.vendor).merge(connection_settings)
228
214
  log_name = device.host.tr(":", "_")
229
215
  if log_directory
230
216
  connection_settings = connection_settings.merge(log_file: File.join(log_directory, "#{log_name}.log"))
231
217
  end
232
218
  connector = @connector_factory.call(device, connection_settings)
219
+ backup, failure = perform_one(connector, backup_class) { yield connector }
220
+ return outcome(device, success_status, backup: backup) unless failure
221
+
222
+ outcome(device, backup ? close_error_status : :failed, backup: backup, error: failure)
223
+ rescue StandardError => error
224
+ outcome(device, :failed, error: error)
225
+ end
226
+
227
+ # 仅信任库定义且产物类型匹配的提交后错误;普通第三方异常上的 backup 字段不代表完成。
228
+ def perform_one(connector, backup_class)
233
229
  backup = nil
234
230
  failure = nil
235
231
  begin
236
- result = yield connector
232
+ result = yield
237
233
  raise TypeError, "backup operation did not return #{backup_class}" unless result.is_a?(backup_class)
238
234
 
239
235
  backup = result
240
236
  rescue StandardError => error
241
237
  failure = error
238
+ if backup_class == Backup && error.instance_of?(BackupPersistenceError) && error.backup.instance_of?(Backup)
239
+ backup = error.backup
240
+ elsif backup_class == TftpBackup && error.instance_of?(TftpCompletionError) && error.transfer.instance_of?(TftpBackup)
241
+ backup = error.transfer
242
+ end
242
243
  ensure
243
244
  begin
244
245
  connector.close
@@ -246,11 +247,7 @@ module Net
246
247
  failure ||= error
247
248
  end
248
249
  end
249
- return outcome(device, success_status, backup: backup) unless failure
250
-
251
- outcome(device, backup ? close_error_status : :failed, backup: backup, error: failure)
252
- rescue StandardError => error
253
- outcome(device, :failed, error: error)
250
+ [backup, failure]
254
251
  end
255
252
  end
256
253
  end
@@ -0,0 +1,49 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Net
4
+ module Connector
5
+ module Netdisco
6
+ # 每次 devices 调用独占一个预算,认证、分页和兼容查询不得重置已用资源或期限。
7
+ class InventoryBudget
8
+ def initialize(options, clock:)
9
+ @options = options
10
+ @clock = clock
11
+ @deadline = @clock.call + options.fetch(:inventory_timeout)
12
+ @bytes = 0
13
+ @devices = 0
14
+ end
15
+
16
+ def remaining
17
+ seconds = @deadline - @clock.call
18
+ raise Client::InventoryTimeout, cause: nil if seconds <= 0
19
+
20
+ seconds
21
+ end
22
+
23
+ # 在追加正文前计数;不能靠 Content-Length 或 JSON 解析后的大小限制内存。
24
+ def consume_bytes(size, response_bytes:)
25
+ remaining
26
+ check_limit!(:max_response_bytes, response_bytes + size)
27
+ check_limit!(:max_inventory_bytes, @bytes + size)
28
+ @bytes += size
29
+ end
30
+
31
+ # 去重前的记录同样占用清单内存;兼容查询也使用这个累计计数。
32
+ def consume_devices(size)
33
+ remaining
34
+ check_limit!(:max_devices, @devices + size)
35
+ @devices += size
36
+ end
37
+
38
+ private
39
+
40
+ def check_limit!(name, value)
41
+ return if value <= @options.fetch(name)
42
+
43
+ raise Client::Error.new("Netdisco inventory exceeded #{name}", code: name), cause: nil
44
+ end
45
+ end
46
+ private_constant :InventoryBudget
47
+ end
48
+ end
49
+ end
@@ -0,0 +1,77 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Net
4
+ module Connector
5
+ module Netdisco
6
+ # 固定一份清单上的选择结果与跳过原因,供预览和执行复用。
7
+ Plan = Data.define(:mode, :inventory, :ready, :outcomes) do
8
+ # Data 只冻结对象自身;这里复制并冻结可能由调用方修改的计划容器。
9
+ def initialize(mode:, inventory:, ready:, outcomes:)
10
+ unless inventory.is_a?(Array) && ready.is_a?(Array) && outcomes.is_a?(Array)
11
+ raise ArgumentError, "plan inventory, ready tasks, and outcomes must be Arrays"
12
+ end
13
+
14
+ tasks = ready.map { |task| task.is_a?(Array) ? task.dup.freeze : task }.freeze
15
+ super(mode: mode, inventory: inventory.dup.freeze, ready: tasks, outcomes: outcomes.dup.freeze)
16
+ end
17
+
18
+ # 按清单顺序返回选中的设备。
19
+ def selected = ready.map(&:last)
20
+
21
+ # 在任何设备 I/O 前校验完整快照,外部构造或修改的计划也必须遵守相同约束。
22
+ def validate!
23
+ raise ArgumentError, "plan mode must be backup or tftp" unless %i[backup tftp].include?(mode)
24
+ unless inventory.all?(Device) && outcomes.size == inventory.size
25
+ raise ArgumentError, "plan inventory and outcomes must have the same device slots"
26
+ end
27
+
28
+ selected_indices = {}
29
+ filenames = {}
30
+ previous_index = -1
31
+ ready.each do |task|
32
+ unless task.is_a?(Array) && task.size == 2 && task.first.is_a?(Integer)
33
+ raise ArgumentError, "plan contains an invalid device task"
34
+ end
35
+
36
+ index, device = task
37
+ unless index > previous_index && index < inventory.size && inventory[index].equal?(device) && device.ready?
38
+ raise ArgumentError, "plan tasks must reference ready inventory devices in order"
39
+ end
40
+ if mode == :tftp
41
+ filename = device.tftp_filename
42
+ raise ArgumentError, "plan cannot upload multiple devices to one TFTP filename" if filenames.key?(filename)
43
+
44
+ filenames[filename] = true
45
+ end
46
+ selected_indices[index] = true
47
+ previous_index = index
48
+ end
49
+
50
+ inventory.each_with_index do |device, index|
51
+ result = outcomes[index]
52
+ next if selected_indices[index] && result.nil?
53
+
54
+ unless !selected_indices[index] && valid_skipped_outcome?(result, device, filenames)
55
+ raise ArgumentError, "plan outcome does not match its inventory slot"
56
+ end
57
+ end
58
+ self
59
+ end
60
+
61
+ private
62
+
63
+ # 跳过结果不携带执行产物;文件冲突必须有本批次实际执行的目标作为依据。
64
+ def valid_skipped_outcome?(result, device, filenames)
65
+ return false unless result.is_a?(Outcome) && result.device.equal?(device) && result.backup.nil? &&
66
+ result.error_code.nil? && result.error_type.nil? &&
67
+ result.started_at.nil? && result.finished_at.nil?
68
+
69
+ return result.status == device.issue unless device.ready?
70
+
71
+ result.status == :sample_limit ||
72
+ (mode == :tftp && result.status == :remote_filename_collision && filenames.key?(device.tftp_filename))
73
+ end
74
+ end
75
+ end
76
+ end
77
+ end