net-connector 0.4.1 → 0.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (107) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +50 -0
  3. data/CONTRIBUTING.md +40 -0
  4. data/README.md +93 -18
  5. data/SECURITY.md +11 -0
  6. data/docs/VERIFICATION.md +139 -6
  7. data/docs/architecture.md +248 -39
  8. data/lib/net/connector/device/base.rb +28 -101
  9. data/lib/net/connector/device/local_backup.rb +96 -0
  10. data/lib/net/connector/device/profile.rb +8 -3
  11. data/lib/net/connector/device/running_config/strategy.rb +1 -1
  12. data/lib/net/connector/device/running_config.rb +46 -5
  13. data/lib/net/connector/device/save_config.rb +21 -0
  14. data/lib/net/connector/device/tftp/file_upload.rb +53 -0
  15. data/lib/net/connector/device/tftp/receipt.rb +71 -0
  16. data/lib/net/connector/device/tftp/strategy.rb +67 -0
  17. data/lib/net/connector/device/tftp/target.rb +63 -0
  18. data/lib/net/connector/device/tftp.rb +117 -0
  19. data/lib/net/connector/device/topology/immediate_strategy.rb +40 -0
  20. data/lib/net/connector/device/topology/interface_description.rb +46 -0
  21. data/lib/net/connector/device/topology/interface_name.rb +58 -0
  22. data/lib/net/connector/device/topology/strategy.rb +75 -0
  23. data/lib/net/connector/device/topology.rb +277 -0
  24. data/lib/net/connector/engine/authentication.rb +1 -1
  25. data/lib/net/connector/engine/command.rb +14 -2
  26. data/lib/net/connector/engine/configuration.rb +11 -8
  27. data/lib/net/connector/engine/dialogue.rb +59 -35
  28. data/lib/net/connector/engine/error_metadata.rb +50 -0
  29. data/lib/net/connector/engine/errors.rb +55 -56
  30. data/lib/net/connector/engine/execution.rb +44 -10
  31. data/lib/net/connector/engine/log/event.rb +66 -0
  32. data/lib/net/connector/engine/log/formatter.rb +16 -0
  33. data/lib/net/connector/engine/log/messages.rb +52 -0
  34. data/lib/net/connector/engine/log/stream.rb +56 -0
  35. data/lib/net/connector/engine/log.rb +107 -84
  36. data/lib/net/connector/engine/session.rb +115 -43
  37. data/lib/net/connector/engine/terminal_renderer.rb +7 -4
  38. data/lib/net/connector/engine/terminal_text.rb +33 -0
  39. data/lib/net/connector/engine/transport.rb +2 -2
  40. data/lib/net/connector/netdisco/batch.rb +25 -33
  41. data/lib/net/connector/netdisco/cli.rb +71 -31
  42. data/lib/net/connector/netdisco/client.rb +206 -73
  43. data/lib/net/connector/netdisco/config_file.rb +26 -4
  44. data/lib/net/connector/netdisco/device.rb +4 -5
  45. data/lib/net/connector/netdisco/diagnostic.rb +61 -0
  46. data/lib/net/connector/netdisco/fleet.rb +96 -66
  47. data/lib/net/connector/netdisco/inventory_budget.rb +49 -0
  48. data/lib/net/connector/netdisco/report.rb +115 -0
  49. data/lib/net/connector/netdisco/result_store.rb +2 -2
  50. data/lib/net/connector/netdisco/rules.rb +28 -5
  51. data/lib/net/connector/netdisco/settings.rb +188 -85
  52. data/lib/net/connector/netdisco/worker.rb +26 -17
  53. data/lib/net/connector/storage/backup_lock.rb +116 -0
  54. data/lib/net/connector/storage/private_file.rb +109 -0
  55. data/lib/net/connector/storage/safe_file.rb +62 -0
  56. data/lib/net/connector/{operations → storage}/saved_config.rb +25 -20
  57. data/lib/net/connector/storage.rb +15 -0
  58. data/lib/net/connector/textfsm.rb +72 -0
  59. data/lib/net/connector/vendor/cisco_ios/tftp_backup.rb +12 -5
  60. data/lib/net/connector/vendor/cisco_ios/topology.rb +12 -6
  61. data/lib/net/connector/vendor/cisco_nxos/tftp_backup.rb +11 -4
  62. data/lib/net/connector/vendor/h3c/tftp_backup.rb +9 -4
  63. data/lib/net/connector/vendor/h3c/topology.rb +11 -6
  64. data/lib/net/connector/vendor/h3c.rb +3 -3
  65. data/lib/net/connector/vendor/hillstone/tftp_backup.rb +15 -6
  66. data/lib/net/connector/vendor/hillstone/topology.rb +11 -6
  67. data/lib/net/connector/vendor/huawei/tftp_backup.rb +9 -3
  68. data/lib/net/connector/vendor/palo_alto/running_config.rb +1 -1
  69. data/lib/net/connector/vendor/palo_alto/tftp_backup.rb +17 -8
  70. data/lib/net/connector/vendor/palo_alto/topology.rb +9 -7
  71. data/lib/net/connector/vendor/radware/tftp_backup.rb +12 -5
  72. data/lib/net/connector/vendor/radware/topology.rb +3 -2
  73. data/lib/net/connector/version.rb +1 -1
  74. data/lib/net/connector.rb +1 -6
  75. metadata +54 -38
  76. data/lib/net/connector/device/interface_description.rb +0 -44
  77. data/lib/net/connector/device/interface_name.rb +0 -56
  78. data/lib/net/connector/engine/base.rb +0 -4
  79. data/lib/net/connector/engine/log_messages.rb +0 -47
  80. data/lib/net/connector/engine/profile.rb +0 -4
  81. data/lib/net/connector/engine.rb +0 -4
  82. data/lib/net/connector/operations/local_backup.rb +0 -49
  83. data/lib/net/connector/operations/parse_output.rb +0 -60
  84. data/lib/net/connector/operations/private_file.rb +0 -25
  85. data/lib/net/connector/operations/running_config/cisco.rb +0 -14
  86. data/lib/net/connector/operations/running_config/hillstone.rb +0 -12
  87. data/lib/net/connector/operations/running_config/palo_alto.rb +0 -12
  88. data/lib/net/connector/operations/running_config/strategy.rb +0 -3
  89. data/lib/net/connector/operations/running_config.rb +0 -16
  90. data/lib/net/connector/operations/tftp/cisco_ios.rb +0 -11
  91. data/lib/net/connector/operations/tftp/cisco_nxos.rb +0 -11
  92. data/lib/net/connector/operations/tftp/file_upload.rb +0 -42
  93. data/lib/net/connector/operations/tftp/h3c.rb +0 -11
  94. data/lib/net/connector/operations/tftp/hillstone.rb +0 -11
  95. data/lib/net/connector/operations/tftp/huawei.rb +0 -11
  96. data/lib/net/connector/operations/tftp/palo_alto.rb +0 -11
  97. data/lib/net/connector/operations/tftp/radware.rb +0 -11
  98. data/lib/net/connector/operations/tftp/strategy.rb +0 -75
  99. data/lib/net/connector/operations/tftp_backup.rb +0 -120
  100. data/lib/net/connector/operations/topology/cisco.rb +0 -11
  101. data/lib/net/connector/operations/topology/h3c.rb +0 -11
  102. data/lib/net/connector/operations/topology/hillstone.rb +0 -11
  103. data/lib/net/connector/operations/topology/palo_alto.rb +0 -11
  104. data/lib/net/connector/operations/topology/radware.rb +0 -11
  105. data/lib/net/connector/operations/topology/strategy.rb +0 -67
  106. data/lib/net/connector/operations/topology.rb +0 -193
  107. data/lib/net/connector/operations.rb +0 -20
@@ -10,7 +10,13 @@ module Net
10
10
  class ConfigFile
11
11
  FIELDS = {
12
12
  "netdisco" => { "url" => ["NETDISCO_URL", :string],
13
- "page_size" => ["NETDISCO_PAGE_SIZE", :integer] },
13
+ "page_size" => ["NETDISCO_PAGE_SIZE", :integer],
14
+ "max_pages" => ["NETDISCO_MAX_PAGES", :integer],
15
+ "max_response_bytes" => ["NETDISCO_MAX_RESPONSE_BYTES", :integer],
16
+ "max_inventory_bytes" => ["NETDISCO_MAX_INVENTORY_BYTES", :integer],
17
+ "max_devices" => ["NETDISCO_MAX_DEVICES", :integer],
18
+ "inventory_timeout" => ["NETDISCO_INVENTORY_TIMEOUT", :number],
19
+ "allow_insecure_http" => ["NETDISCO_ALLOW_INSECURE_HTTP", :boolean] },
14
20
  "backup" => { "directory" => ["NET_CONNECTOR_BACKUP_DIRECTORY", :string],
15
21
  "concurrency" => ["NET_CONNECTOR_CONCURRENCY", :integer],
16
22
  "limit_per_vendor" => ["NET_CONNECTOR_SAMPLE_PER_VENDOR", :integer] },
@@ -21,12 +27,16 @@ module Net
21
27
  "host_overrides" => ["NET_CONNECTOR_HOST_OVERRIDES", :json],
22
28
  "device_rules" => ["NET_CONNECTOR_DEVICE_RULES", :json] },
23
29
  "ssh" => { "protocol" => ["NET_CONNECTOR_PROTOCOL", :string],
30
+ "max_script_output_bytes" => ["NET_CONNECTOR_MAX_SCRIPT_OUTPUT_BYTES", :optional_integer],
24
31
  "known_hosts" => ["NET_CONNECTOR_KNOWN_HOSTS", :string],
25
32
  "host_key_policy" => ["NET_CONNECTOR_HOST_KEY_POLICY", :string],
26
33
  "log_directory" => ["NET_CONNECTOR_LOG_DIRECTORY", :string],
27
34
  "log_level" => ["NET_CONNECTOR_LOG_LEVEL", :string] },
28
35
  "tftp" => { "server" => ["TFTP_HOST", :string],
29
- "vrfs" => ["NET_CONNECTOR_TFTP_VRFS", :json] }
36
+ "vrfs" => ["NET_CONNECTOR_TFTP_VRFS", :json],
37
+ "h3c_source_file" => ["NET_CONNECTOR_H3C_TFTP_SOURCE_FILE", :string],
38
+ "h3c_wireless_source_file" => ["NET_CONNECTOR_H3C_WIRELESS_TFTP_SOURCE_FILE", :string],
39
+ "huawei_source_file" => ["NET_CONNECTOR_HUAWEI_TFTP_SOURCE_FILE", :string] }
30
40
  }.freeze
31
41
 
32
42
  # 安全读取 YAML 配置并映射到环境变量格式。
@@ -52,7 +62,7 @@ module Net
52
62
  end
53
63
  end
54
64
  rescue Psych::Exception => error
55
- raise ArgumentError, "invalid YAML config (#{error.class})"
65
+ raise ArgumentError, "invalid YAML config (#{error.class})", cause: nil
56
66
  end
57
67
 
58
68
  # 按字段类型校验并编码单项配置。
@@ -62,9 +72,21 @@ module Net
62
72
  raise ArgumentError, "#{section}.#{name} must be a nonempty String" unless value.is_a?(String) && !value.strip.empty?
63
73
 
64
74
  value
65
- when :integer
75
+ when :integer, :optional_integer
76
+ return nil if type == :optional_integer && value.nil?
77
+
66
78
  raise ArgumentError, "#{section}.#{name} must be a positive Integer" unless value.is_a?(Integer) && value.positive?
67
79
 
80
+ value.to_s
81
+ when :number
82
+ unless value.is_a?(Numeric) && value.real? && value.finite? && value.positive? && value.to_f.finite?
83
+ raise ArgumentError, "#{section}.#{name} must be a positive finite number"
84
+ end
85
+
86
+ value.to_s
87
+ when :boolean
88
+ raise ArgumentError, "#{section}.#{name} must be true or false" unless [true, false].include?(value)
89
+
68
90
  value.to_s
69
91
  when :list
70
92
  unless value.is_a?(Array) && value.all? { |item| item.is_a?(String) && !item.empty? && !item.include?(",") }
@@ -1,7 +1,7 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  require "ipaddr"
4
- require_relative "../operations/saved_config"
4
+ require_relative "../storage/saved_config"
5
5
 
6
6
  module Net
7
7
  module Connector
@@ -44,7 +44,7 @@ module Net
44
44
  def ready? = issue.nil?
45
45
 
46
46
  # 生成不随名称或厂商信息改变的本地文本文件名。
47
- def backup_filename = Operations::SavedConfig.filename(host)
47
+ def backup_filename = Storage::SavedConfig.filename(host)
48
48
 
49
49
  # 生成设备 CLI 可用的 ASCII 远端文件名。
50
50
  def tftp_filename
@@ -53,13 +53,12 @@ module Net
53
53
  .gsub(/\A[._-]+|[._-]+\z/, "")
54
54
  label = vendor&.to_s || "device" if label.empty?
55
55
  strategy = Net::Connector.vendor_class(vendor).profile.tftp_strategy if vendor
56
- # 旧版自定义策略只需实现实例接口,缺少命名接口时沿用通用 cfg 名称。
57
- strategy = Operations::Tftp::Strategy unless strategy.respond_to?(:filename)
56
+ raise UnsupportedOperation, "device has no TFTP strategy" unless strategy
58
57
  TftpTarget.validate_path!(strategy.filename(host, label: label))
59
58
  end
60
59
 
61
60
  # 按厂商及设备地址实例化连接器。
62
- def connector(**settings)
61
+ def build_connector(**settings)
63
62
  raise ArgumentError, "device is not ready: #{issue}" unless ready?
64
63
  raise ArgumentError, "host cannot be overridden" if settings.key?(:host)
65
64
 
@@ -0,0 +1,61 @@
1
+ # frozen_string_literal: true
2
+
3
+ require_relative "../engine/error_metadata"
4
+
5
+ module Net
6
+ module Connector
7
+ module Netdisco
8
+ # 报告是新的诊断通道:只接受固定词表,不展开异常消息、调用栈、命令、source 或 line。
9
+ Diagnostic = Data.define(:error_code, :error_type, :phase, :underlying_type,
10
+ :artifact_state, :artifact_phase, :verification)
11
+
12
+ class Diagnostic
13
+ ARTIFACT_STATES = %i[not_committed committed durable reported_uploaded].freeze
14
+ ARTIFACT_PHASES = %i[resolve_parent temporary_file write file_sync rename directory_open directory_sync
15
+ cleanup complete path finalize].freeze
16
+ private_constant :ARTIFACT_STATES, :ARTIFACT_PHASES
17
+
18
+ # 包括手工构造的结果在内,v2 都不能携带自由文本诊断值。
19
+ def initialize(error_code: nil, error_type: nil, phase: nil, underlying_type: nil,
20
+ artifact_state: nil, artifact_phase: nil, verification: nil)
21
+ super(error_code: ErrorMetadata.code(error_code),
22
+ error_type: ErrorMetadata.type(error_type), phase: ErrorMetadata.phase(phase),
23
+ underlying_type: ErrorMetadata.type(underlying_type),
24
+ artifact_state: ARTIFACT_STATES.include?(artifact_state) ? artifact_state : nil,
25
+ artifact_phase: ARTIFACT_PHASES.include?(artifact_phase) ? artifact_phase : nil,
26
+ verification: verification == :device_reported ? verification : nil)
27
+ end
28
+
29
+ # 读取已知错误的白名单属性后立即丢弃异常引用,不把回调对象保留进批次。
30
+ def self.from(error, backup: nil, phase: nil)
31
+ return unless error
32
+
33
+ fields = { error_type: ErrorMetadata.type(error.class.name) || "StandardError", phase: phase }
34
+ if error.is_a?(Net::Connector::Error)
35
+ fields.merge!(error_code: error.code, phase: phase || error.phase)
36
+ fields[:underlying_type] = error.underlying.type if error.underlying.instance_of?(UnderlyingError)
37
+ end
38
+ new(**fields.merge(completion_fields(error, backup, phase)))
39
+ rescue StandardError
40
+ new(error_type: "StandardError", phase: phase)
41
+ end
42
+
43
+ def self.completion_fields(error, backup, phase)
44
+ if error.instance_of?(BackupPersistenceError) && backup.instance_of?(Backup) && error.backup.equal?(backup)
45
+ { artifact_state: error.receipt.state, artifact_phase: error.receipt.phase,
46
+ underlying_type: error.underlying_type }
47
+ elsif error.instance_of?(TftpCompletionError) && backup.instance_of?(TftpReceipt) && error.receipt.equal?(backup)
48
+ { artifact_state: :reported_uploaded, artifact_phase: error.code == :transfer_finalize_failed ? :finalize : :path,
49
+ verification: :device_reported, underlying_type: error.underlying_type }
50
+ elsif phase == :report && Storage::PrivateFile.receipt_error?(error)
51
+ { error_code: error.code, artifact_state: error.receipt.state, artifact_phase: error.receipt.phase,
52
+ underlying_type: error.underlying_type }
53
+ else
54
+ {}
55
+ end
56
+ end
57
+ private_class_method :completion_fields
58
+ end
59
+ end
60
+ end
61
+ end
@@ -8,101 +8,130 @@ module Net
8
8
  module Netdisco
9
9
  # 基于已验证的清单快照组织有上限的并发备份。
10
10
  class Fleet
11
+ DEFAULT_SETTING = Object.new.freeze
12
+ private_constant :DEFAULT_SETTING
13
+
11
14
  # 组装清单客户端、规则、凭据与结果存储。
12
15
  def initialize(client: nil, settings: Settings.new, rules: nil, credentials: nil, connector_factory: nil,
13
16
  result_store: ResultStore::Text.new)
14
17
  @settings = settings
15
- @client = client || settings.client
16
- @rules = rules || settings.rules
17
- @credentials = credentials || settings.method(:credentials_for)
18
- @connector_factory = connector_factory || ->(device, connection_settings) { device.connector(**connection_settings) }
18
+ @client = client
19
+ @rules = rules
20
+ @credentials = credentials
21
+ @connector_factory = connector_factory || ->(device, connection_settings) { device.build_connector(**connection_settings) }
19
22
  @result_store = result_store
20
- raise ArgumentError, "credentials must respond to call" unless @credentials.respond_to?(:call)
23
+ raise ArgumentError, "credentials must respond to call" if @credentials && !@credentials.respond_to?(:call)
21
24
  raise ArgumentError, "result_store must respond to write" if result_store && !result_store.respond_to?(:write)
22
25
  end
23
26
 
24
27
  # 拉取并标记重复管理地址的设备清单。
25
28
  def devices
26
- rows = @client.devices
27
- raise Client::Error, "Netdisco returned an invalid device inventory" unless rows.is_a?(Array)
28
-
29
- devices = rows.map { |row| Device.from_row(row, rules: @rules) }
30
- duplicates = devices.group_by(&:host).reject { |host, entries| host.nil? || entries.size == 1 }
31
- devices.map do |device|
32
- duplicates.key?(device.host) ? device.with(issue: :duplicate_host) : device
33
- end.freeze
29
+ inventory(@settings.snapshot(mode: :inventory))
34
30
  end
35
31
 
36
32
  # 规划本地配置备份的设备集合。
37
33
  def plan_backup(limit_per_vendor: nil)
38
- Planner.new(devices).call(mode: :backup, limit_per_vendor: limit_per_vendor)
34
+ build_plan(:backup, limit_per_vendor, @settings.snapshot(mode: :backup))
39
35
  end
40
36
 
41
37
  # 规划 TFTP 配置备份的设备集合。
42
38
  def plan_tftp_backup(limit_per_vendor: 5)
43
- Planner.new(devices).call(mode: :tftp, limit_per_vendor: limit_per_vendor)
39
+ build_plan(:tftp, limit_per_vendor, @settings.snapshot(mode: :tftp))
44
40
  end
45
41
 
46
42
  # 先拉取完整清单,再并发采集并保存设备配置。
47
- def backup_all(directory: @settings.backup_directory, concurrency: @settings.concurrency,
48
- limit_per_vendor: nil, plan: nil, on_start: nil, on_result: nil, on_change: nil)
43
+ def backup_all(directory: DEFAULT_SETTING, concurrency: DEFAULT_SETTING,
44
+ limit_per_vendor: nil, plan: nil, on_start: nil, on_result: nil, on_change: nil,
45
+ success_policy: :strict)
46
+ reporting = Report.validate_policy!(success_policy)
47
+ policy = @settings.snapshot(mode: :backup)
48
+ directory = policy.backup_directory if directory.equal?(DEFAULT_SETTING)
49
+ concurrency = policy.concurrency if concurrency.equal?(DEFAULT_SETTING)
49
50
  raise ArgumentError, "directory must be a nonempty String" unless directory.is_a?(String) && !directory.empty?
51
+ Worker.new(concurrency: concurrency)
50
52
  Planner.validate_limit!(limit_per_vendor)
51
53
  validate_callback!(on_result, :on_result)
52
54
  validate_callback!(on_start, :on_start)
53
55
  validate_callback!(on_change, :on_change)
54
56
 
55
- plan ||= plan_backup(limit_per_vendor: limit_per_vendor)
57
+ plan ||= build_plan(:backup, limit_per_vendor, policy)
56
58
  validate_plan!(plan, :backup)
57
59
  callbacks = [on_result]
58
60
  callbacks << ->(item) { on_change.call(item) if item.backup.is_a?(Backup) && item.backup.changed? } if on_change
59
61
  target_directory = File.expand_path(directory)
60
62
  run_batch(:backup, plan, concurrency: concurrency, report_directory: directory,
61
- output_directory: target_directory, on_start: on_start, on_result: callbacks.compact) do |device, log_directory|
62
- backup_one(device, target_directory, log_directory)
63
+ output_directory: target_directory, policy: policy, reporting: reporting,
64
+ on_start: on_start, on_result: callbacks.compact) do |device, log_directory|
65
+ backup_one(device, target_directory, log_directory, policy)
63
66
  end
64
67
  end
65
68
 
66
69
  # 让设备主动导出配置,在执行前检查同批目标文件名冲突。
67
- def tftp_backup_all(server:, source_files: {}, concurrency: @settings.concurrency,
70
+ def tftp_backup_all(server:, source_files: {}, concurrency: DEFAULT_SETTING,
68
71
  limit_per_vendor: 5, vrfs: {}, on_start: nil, on_result: nil, plan: nil,
69
- report_directory: @settings.backup_directory)
70
- TftpTarget.new(host: server, path: "preflight.cfg")
72
+ report_directory: DEFAULT_SETTING, success_policy: :strict)
73
+ reporting = Report.validate_policy!(success_policy)
74
+ policy = @settings.snapshot(mode: :tftp)
75
+ concurrency = policy.concurrency if concurrency.equal?(DEFAULT_SETTING)
76
+ report_directory = policy.backup_directory if report_directory.equal?(DEFAULT_SETTING)
77
+ Worker.new(concurrency: concurrency)
78
+ server = TftpTarget.new(host: server, path: "preflight.cfg").host
71
79
  raise ArgumentError, "source_files must be a Hash" unless source_files.is_a?(Hash)
72
80
  source_files.each_value { |source_file| TftpTarget.validate_source_file!(source_file) }
73
- validate_vrfs!(vrfs)
81
+ source_files = source_files.transform_values { |value| value.dup.freeze }.freeze
82
+ vrfs = Settings.validate_vrfs!(vrfs).transform_values { |value| value.dup.freeze }.freeze
74
83
  Planner.validate_limit!(limit_per_vendor)
75
84
  validate_callback!(on_result, :on_result)
76
85
  validate_callback!(on_start, :on_start)
77
86
 
78
- plan ||= plan_tftp_backup(limit_per_vendor: limit_per_vendor)
87
+ plan ||= build_plan(:tftp, limit_per_vendor, policy)
79
88
  validate_plan!(plan, :tftp)
80
89
  run_batch(:tftp, plan, concurrency: concurrency, report_directory: report_directory,
81
- on_start: on_start, on_result: on_result) do |device, log_directory|
82
- tftp_backup_one(device, server, source_files, log_directory, vrfs)
90
+ policy: policy, reporting: reporting, on_start: on_start, on_result: on_result) do |device, log_directory|
91
+ tftp_backup_one(device, server, source_files, log_directory, vrfs, policy)
83
92
  end
84
93
  end
85
94
 
86
95
  private
87
96
 
97
+ # 策略在清单请求之前固定;注入的清单客户端仍由调用方负责其自身生命周期。
98
+ def inventory(policy)
99
+ rows = (@client || @settings.client(policy: policy)).devices
100
+ raise Client::Error, "Netdisco returned an invalid device inventory" unless rows.is_a?(Array)
101
+
102
+ rules = @rules || policy.rules
103
+ devices = rows.map { |row| Device.from_row(row, rules: rules) }
104
+ duplicates = devices.group_by(&:host).reject { |host, entries| host.nil? || entries.size == 1 }
105
+ devices.map { |device| duplicates.key?(device.host) ? device.with(issue: :duplicate_host) : device }.freeze
106
+ end
107
+
108
+ def build_plan(mode, limit, policy)
109
+ Planner.validate_limit!(limit)
110
+ Planner.new(inventory(policy)).call(mode: mode, limit_per_vendor: limit)
111
+ end
112
+
88
113
  # 统一组织设备任务、日志目录与报告;Worker 保证单台设备失败不影响其他设备。
89
- def run_batch(mode, plan, concurrency:, report_directory:, on_start:, on_result:, output_directory: nil)
114
+ def run_batch(mode, plan, concurrency:, report_directory:, policy:, reporting:, on_start:, on_result:, output_directory: nil)
90
115
  worker = Worker.new(concurrency: concurrency)
91
116
  started_at = Time.now.utc
117
+ started = monotonic
92
118
  outcomes = plan.outcomes.dup
93
119
  FileUtils.mkdir_p(output_directory, mode: 0o700) if output_directory && !plan.ready.empty?
94
- log_directory = @settings.log_directory
120
+ log_directory = policy.log_directory
95
121
  FileUtils.mkdir_p(log_directory, mode: 0o700) if log_directory && !plan.ready.empty?
96
122
  callback_errors = worker.run(plan.ready, outcomes: outcomes, on_start: on_start, on_result: on_result,
97
- on_error: ->(device, error) { outcome(device, :failed, error: error) }) do |device|
123
+ on_error: ->(device, error) { build_outcome(device, :failed, error: error) }) do |device|
98
124
  yield device, log_directory
99
125
  end
100
126
  batch = Batch.new(mode: mode, outcomes: outcomes.freeze, started_at: started_at,
101
127
  finished_at: Time.now.utc, callback_errors: callback_errors,
102
128
  report_location: nil, report_error: nil)
129
+ batch = Report.new(batch, policy: reporting, duration_ms: ((monotonic - started) * 1000).round)
103
130
  save_report(batch, directory: report_directory)
104
131
  end
105
132
 
133
+ def monotonic = Process.clock_gettime(Process::CLOCK_MONOTONIC)
134
+
106
135
  # 校验批量任务回调接口。
107
136
  def validate_callback!(callback, name)
108
137
  raise ArgumentError, "#{name} must respond to call" if callback && !callback.respond_to?(:call)
@@ -113,18 +142,12 @@ module Net
113
142
  return batch unless @result_store
114
143
 
115
144
  batch.with(report_location: @result_store.write(batch, directory: directory))
145
+ rescue Storage::PrivateFile::WriteError => error
146
+ # 报告也可能已经原子替换;保留已知位置,不将持久性失败误写成完全没有产物。
147
+ location = error.receipt.path if Storage::PrivateFile.receipt_error?(error) && error.receipt.committed?
148
+ batch.with_report_error(error, location: location)
116
149
  rescue StandardError => error
117
- batch.with(report_error: error.class.name)
118
- end
119
-
120
- # 校验厂商 VRF 映射和名称。
121
- def validate_vrfs!(vrfs)
122
- unless vrfs.is_a?(Hash) && vrfs.all? { |vendor, value|
123
- %i[cisco_nxos hillstone].include?(vendor) && value.is_a?(String) &&
124
- value.match?(/\A[A-Za-z0-9_][A-Za-z0-9_.-]*\z/)
125
- }
126
- raise ArgumentError, "vrfs must map supported vendor names to safe VRF names"
127
- end
150
+ batch.with_report_error(error)
128
151
  end
129
152
 
130
153
  # 确认传入计划属于当前备份模式。
@@ -135,16 +158,16 @@ module Net
135
158
  end
136
159
 
137
160
  # 构造包含错误类型的单台设备结果。
138
- def outcome(device, status, backup: nil, error: nil)
161
+ def build_outcome(device, status, backup: nil, error: nil)
139
162
  Outcome.new(device: device, status: status, backup: backup,
140
163
  error_code: error.respond_to?(:code) ? error.code : nil,
141
- error_type: error&.class&.name)
164
+ error_type: error&.class&.name, diagnostic: Diagnostic.from(error, backup: backup))
142
165
  end
143
166
 
144
167
  # 执行单台设备的 TFTP 导出。
145
- def tftp_backup_one(device, server, source_files, log_directory, vrfs)
146
- run_one(device, log_directory, success_status: :reported_uploaded,
147
- close_error_status: :reported_with_error, backup_class: TftpBackup) do |connector|
168
+ def tftp_backup_one(device, server, source_files, log_directory, vrfs, policy)
169
+ run_one(device, log_directory, policy, success_status: :reported_uploaded,
170
+ close_error_status: :reported_with_error, backup_class: TftpReceipt) do |connector|
148
171
  source_file = source_files.fetch(device.vendor, nil)
149
172
  transfer_settings = { host: server, path: device.tftp_filename, source_file: source_file }
150
173
  transfer_settings[:vrf] = vrfs.fetch(device.vendor) if vrfs.key?(device.vendor)
@@ -153,46 +176,57 @@ module Net
153
176
  end
154
177
 
155
178
  # 执行单台设备的本地配置采集。
156
- def backup_one(device, directory, log_directory)
179
+ def backup_one(device, directory, log_directory, policy)
157
180
  destination = File.join(directory, device.backup_filename)
158
- previous = Operations::SavedConfig.new(directory: directory).find(device.host, required: false)
159
- previous_digest = Digest::SHA256.file(previous).hexdigest if previous && previous != destination
160
- run_one(device, log_directory, success_status: :backed_up,
161
- close_error_status: :saved_with_error, backup_class: Backup) do |connector|
162
- backup = connector.backup(path: destination)
163
- if backup.is_a?(Backup) && backup.change == :created && previous_digest
164
- backup.with(previous_sha256: previous_digest,
165
- change: backup.sha256 == previous_digest ? :unchanged : :changed)
166
- else
167
- backup
181
+ Storage::BackupLock.synchronize(destination, host: device.host) do |path_lock|
182
+ Storage::SafeFile.open(destination, missing: true) { |_file, _stat| nil }
183
+ run_one(device, log_directory, policy, success_status: :backed_up,
184
+ close_error_status: :saved_with_error, backup_class: Backup) do |connector|
185
+ path_lock.with_delegated_lock { connector.backup(path: destination) }
168
186
  end
169
187
  end
170
188
  end
171
189
 
172
190
  # 隔离单台设备的凭据、连接、操作和关闭异常。
173
- def run_one(device, log_directory, success_status:, close_error_status:, backup_class:)
174
- connection_settings = @credentials.call(device)
175
- return outcome(device, :missing_credentials) if connection_settings.nil?
191
+ def run_one(device, log_directory, policy, success_status:, close_error_status:, backup_class:)
192
+ connection_settings = @credentials ? @credentials.call(device) : @settings.device_credentials_for(device)
193
+ return build_outcome(device, :missing_credentials) if connection_settings.nil?
176
194
  unless connection_settings.is_a?(Hash) && connection_settings.keys.all?(Symbol) &&
177
195
  connection_settings[:username].is_a?(String) && !connection_settings[:username].empty? &&
178
196
  !connection_settings.key?(:host)
179
197
  raise ArgumentError, "credential resolver must return settings with username and without host"
180
198
  end
181
199
 
200
+ connection_settings = policy.connection_options(device.vendor).merge(connection_settings)
182
201
  log_name = device.host.tr(":", "_")
183
202
  if log_directory
184
203
  connection_settings = connection_settings.merge(log_file: File.join(log_directory, "#{log_name}.log"))
185
204
  end
186
205
  connector = @connector_factory.call(device, connection_settings)
206
+ backup, failure = perform_one(connector, backup_class) { yield connector }
207
+ return build_outcome(device, success_status, backup: backup) unless failure
208
+
209
+ build_outcome(device, backup ? close_error_status : :failed, backup: backup, error: failure)
210
+ rescue StandardError => error
211
+ build_outcome(device, :failed, error: error)
212
+ end
213
+
214
+ # 仅信任库定义且产物类型匹配的提交后错误;普通第三方异常上的 backup 字段不代表完成。
215
+ def perform_one(connector, backup_class)
187
216
  backup = nil
188
217
  failure = nil
189
218
  begin
190
- result = yield connector
219
+ result = yield
191
220
  raise TypeError, "backup operation did not return #{backup_class}" unless result.is_a?(backup_class)
192
221
 
193
222
  backup = result
194
223
  rescue StandardError => error
195
224
  failure = error
225
+ if backup_class == Backup && error.instance_of?(BackupPersistenceError) && error.backup.instance_of?(Backup)
226
+ backup = error.backup
227
+ elsif backup_class == TftpReceipt && error.instance_of?(TftpCompletionError) && error.receipt.instance_of?(TftpReceipt)
228
+ backup = error.receipt
229
+ end
196
230
  ensure
197
231
  begin
198
232
  connector.close
@@ -200,11 +234,7 @@ module Net
200
234
  failure ||= error
201
235
  end
202
236
  end
203
- return outcome(device, success_status, backup: backup) unless failure
204
-
205
- outcome(device, backup ? close_error_status : :failed, backup: backup, error: failure)
206
- rescue StandardError => error
207
- outcome(device, :failed, error: error)
237
+ [backup, failure]
208
238
  end
209
239
  end
210
240
  end
@@ -0,0 +1,49 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Net
4
+ module Connector
5
+ module Netdisco
6
+ # 每次 devices 调用独占一个预算,认证、分页和兼容查询不得重置已用资源或期限。
7
+ class InventoryBudget
8
+ def initialize(options, clock:)
9
+ @options = options
10
+ @clock = clock
11
+ @deadline = @clock.call + options.fetch(:inventory_timeout)
12
+ @bytes = 0
13
+ @devices = 0
14
+ end
15
+
16
+ def remaining
17
+ seconds = @deadline - @clock.call
18
+ raise Client::InventoryTimeout, cause: nil if seconds <= 0
19
+
20
+ seconds
21
+ end
22
+
23
+ # 在追加正文前计数;不能靠 Content-Length 或 JSON 解析后的大小限制内存。
24
+ def consume_bytes(size, response_bytes:)
25
+ remaining
26
+ check_limit!(:max_response_bytes, response_bytes + size)
27
+ check_limit!(:max_inventory_bytes, @bytes + size)
28
+ @bytes += size
29
+ end
30
+
31
+ # 去重前的记录同样占用清单内存;兼容查询也使用这个累计计数。
32
+ def consume_devices(size)
33
+ remaining
34
+ check_limit!(:max_devices, @devices + size)
35
+ @devices += size
36
+ end
37
+
38
+ private
39
+
40
+ def check_limit!(name, value)
41
+ return if value <= @options.fetch(name)
42
+
43
+ raise Client::Error.new("Netdisco inventory exceeded #{name}", code: name), cause: nil
44
+ end
45
+ end
46
+ private_constant :InventoryBudget
47
+ end
48
+ end
49
+ end
@@ -0,0 +1,115 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "forwardable"
4
+ require_relative "batch"
5
+
6
+ module Net
7
+ module Connector
8
+ module Netdisco
9
+ # 唯一的批量报告契约,分开呈现成功策略、清单覆盖和受控诊断。
10
+ class Report
11
+ extend Forwardable
12
+ SUCCESS = %i[backed_up reported_uploaded].freeze
13
+ ATTEMPTED = [*SUCCESS, :failed, :saved_with_error, :reported_with_error].freeze
14
+ IGNORABLE = %i[filtered sample_limit].freeze
15
+ private_constant :SUCCESS, :ATTEMPTED, :IGNORABLE
16
+
17
+ attr_reader :batch, :policy, :duration_ms, :report_diagnostic
18
+ def_delegators :@batch, *Batch.members, :success?, :status, :counts
19
+
20
+ def self.validate_policy!(policy)
21
+ raise ArgumentError, "success_policy must be strict or selected" unless %i[strict selected].include?(policy)
22
+
23
+ policy
24
+ end
25
+
26
+ def initialize(batch, policy: :strict, duration_ms: nil, report_diagnostic: nil)
27
+ self.class.validate_policy!(policy)
28
+ raise ArgumentError, "report requires a Batch" unless batch.instance_of?(Batch)
29
+ unless duration_ms.nil? || (duration_ms.is_a?(Numeric) && duration_ms.real? && duration_ms.finite? && duration_ms >= 0)
30
+ raise ArgumentError, "duration_ms must be nonnegative and finite"
31
+ end
32
+ unless report_diagnostic.nil? || report_diagnostic.instance_of?(Diagnostic)
33
+ raise ArgumentError, "report_diagnostic must be a Diagnostic"
34
+ end
35
+
36
+ @batch, @policy, @duration_ms, @report_diagnostic = batch, policy, duration_ms, report_diagnostic
37
+ freeze
38
+ end
39
+
40
+ # 严格完成情况与所选成功策略分别保留,未知状态不能当作可忽略跳过。
41
+ def policy_success?
42
+ return batch.success? if policy == :strict
43
+ return false unless callback_errors.empty? && report_error.nil?
44
+
45
+ outcomes.any? { |item| SUCCESS.include?(item.status) } &&
46
+ outcomes.all? { |item| SUCCESS.include?(item.status) || IGNORABLE.include?(item.status) }
47
+ end
48
+
49
+ def with(**attributes)
50
+ self.class.new(batch.with(**attributes), policy: policy, duration_ms: duration_ms,
51
+ report_diagnostic: report_diagnostic)
52
+ end
53
+
54
+ def with_report_error(error, location: nil)
55
+ diagnostic = Diagnostic.from(error, phase: :report)
56
+ self.class.new(batch.with(report_location: location, report_error: diagnostic.error_type),
57
+ policy: policy, duration_ms: duration_ms, report_diagnostic: diagnostic)
58
+ end
59
+
60
+ # 手工构造的批次同样经过诊断白名单,不序列化任意异常内容。
61
+ def summary
62
+ data = batch_summary
63
+ data.merge(
64
+ schema_version: 2, policy: policy, policy_success: policy_success?, duration_ms: duration_ms,
65
+ coverage: { complete: !outcomes.empty? && outcomes.all? { |item| ATTEMPTED.include?(item.status) },
66
+ attempted: outcomes.count { |item| ATTEMPTED.include?(item.status) },
67
+ skipped: outcomes.count { |item| !ATTEMPTED.include?(item.status) } },
68
+ devices: device_summaries(data.fetch(:devices)),
69
+ callback_errors: callback_errors.map { |entry| { host: entry[:host], error_type: ErrorMetadata.type(entry[:error_type]) } },
70
+ report_location: report_location, report_error: ErrorMetadata.type(report_error),
71
+ report_diagnostic: report_diagnostic&.to_h
72
+ )
73
+ end
74
+
75
+ def inspect = "#<#{self.class} policy=#{policy} status=#{status} policy_success=#{policy_success?}>"
76
+
77
+ private
78
+
79
+ # 生成可写入报告及供命令行展示的结构化摘要。
80
+ def batch_summary
81
+ {
82
+ mode: mode, status: status, started_at: started_at.iso8601, finished_at: finished_at.iso8601,
83
+ total: outcomes.size,
84
+ succeeded: outcomes.count(&:success?),
85
+ partial: outcomes.count(&:partial?),
86
+ failed: outcomes.count { |item| item.status == :failed },
87
+ skipped: outcomes.count(&:skipped?),
88
+ counts: counts,
89
+ callback_errors: callback_errors,
90
+ devices: outcomes.map do |item|
91
+ { host: item.device.host || item.device.source_ip, name: item.device.name,
92
+ vendor: item.device.vendor,
93
+ status: item.status, path: item.backup&.path,
94
+ bytes: item.backup.is_a?(Backup) ? item.backup.bytes : nil,
95
+ sha256: item.backup.is_a?(Backup) ? item.backup.sha256 : nil,
96
+ change: item.backup.is_a?(Backup) ? item.backup.change : nil,
97
+ previous_sha256: item.backup.is_a?(Backup) ? item.backup.previous_sha256 : nil,
98
+ started_at: item.started_at&.iso8601, finished_at: item.finished_at&.iso8601,
99
+ duration_ms: item.duration_ms,
100
+ error_code: item.error_code, error_type: item.error_type }
101
+ end
102
+ }
103
+ end
104
+
105
+ def device_summaries(entries)
106
+ entries.zip(outcomes).map do |entry, item|
107
+ diagnostic = item.diagnostic || Diagnostic.new(error_code: item.error_code, error_type: item.error_type)
108
+ entry.merge(error_code: diagnostic.error_code, error_type: diagnostic.error_type,
109
+ diagnostic: diagnostic.to_h.except(:error_code, :error_type))
110
+ end
111
+ end
112
+ end
113
+ end
114
+ end
115
+ end
@@ -3,7 +3,7 @@
3
3
  require "json"
4
4
  require "fileutils"
5
5
  require "securerandom"
6
- require_relative "../operations/private_file"
6
+ require_relative "../storage/private_file"
7
7
 
8
8
  module Net
9
9
  module Connector
@@ -17,7 +17,7 @@ module Net
17
17
  FileUtils.mkdir_p(destination, mode: 0o700)
18
18
  filename = "netdisco-#{batch.mode}-#{batch.started_at.strftime("%Y%m%dT%H%M%SZ")}-#{SecureRandom.hex(4)}.json"
19
19
  path = File.join(destination, filename)
20
- Operations::PrivateFile.write(path, JSON.pretty_generate(batch.summary))
20
+ Storage::PrivateFile.write(path, JSON.pretty_generate(batch.summary)).path
21
21
  end
22
22
  end
23
23