net-connector 0.4.1 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +50 -0
- data/CONTRIBUTING.md +40 -0
- data/README.md +93 -18
- data/SECURITY.md +11 -0
- data/docs/VERIFICATION.md +139 -6
- data/docs/architecture.md +248 -39
- data/lib/net/connector/device/base.rb +28 -101
- data/lib/net/connector/device/local_backup.rb +96 -0
- data/lib/net/connector/device/profile.rb +8 -3
- data/lib/net/connector/device/running_config/strategy.rb +1 -1
- data/lib/net/connector/device/running_config.rb +46 -5
- data/lib/net/connector/device/save_config.rb +21 -0
- data/lib/net/connector/device/tftp/file_upload.rb +53 -0
- data/lib/net/connector/device/tftp/receipt.rb +71 -0
- data/lib/net/connector/device/tftp/strategy.rb +67 -0
- data/lib/net/connector/device/tftp/target.rb +63 -0
- data/lib/net/connector/device/tftp.rb +117 -0
- data/lib/net/connector/device/topology/immediate_strategy.rb +40 -0
- data/lib/net/connector/device/topology/interface_description.rb +46 -0
- data/lib/net/connector/device/topology/interface_name.rb +58 -0
- data/lib/net/connector/device/topology/strategy.rb +75 -0
- data/lib/net/connector/device/topology.rb +277 -0
- data/lib/net/connector/engine/authentication.rb +1 -1
- data/lib/net/connector/engine/command.rb +14 -2
- data/lib/net/connector/engine/configuration.rb +11 -8
- data/lib/net/connector/engine/dialogue.rb +59 -35
- data/lib/net/connector/engine/error_metadata.rb +50 -0
- data/lib/net/connector/engine/errors.rb +55 -56
- data/lib/net/connector/engine/execution.rb +44 -10
- data/lib/net/connector/engine/log/event.rb +66 -0
- data/lib/net/connector/engine/log/formatter.rb +16 -0
- data/lib/net/connector/engine/log/messages.rb +52 -0
- data/lib/net/connector/engine/log/stream.rb +56 -0
- data/lib/net/connector/engine/log.rb +107 -84
- data/lib/net/connector/engine/session.rb +115 -43
- data/lib/net/connector/engine/terminal_renderer.rb +7 -4
- data/lib/net/connector/engine/terminal_text.rb +33 -0
- data/lib/net/connector/engine/transport.rb +2 -2
- data/lib/net/connector/netdisco/batch.rb +25 -33
- data/lib/net/connector/netdisco/cli.rb +71 -31
- data/lib/net/connector/netdisco/client.rb +206 -73
- data/lib/net/connector/netdisco/config_file.rb +26 -4
- data/lib/net/connector/netdisco/device.rb +4 -5
- data/lib/net/connector/netdisco/diagnostic.rb +61 -0
- data/lib/net/connector/netdisco/fleet.rb +96 -66
- data/lib/net/connector/netdisco/inventory_budget.rb +49 -0
- data/lib/net/connector/netdisco/report.rb +115 -0
- data/lib/net/connector/netdisco/result_store.rb +2 -2
- data/lib/net/connector/netdisco/rules.rb +28 -5
- data/lib/net/connector/netdisco/settings.rb +188 -85
- data/lib/net/connector/netdisco/worker.rb +26 -17
- data/lib/net/connector/storage/backup_lock.rb +116 -0
- data/lib/net/connector/storage/private_file.rb +109 -0
- data/lib/net/connector/storage/safe_file.rb +62 -0
- data/lib/net/connector/{operations → storage}/saved_config.rb +25 -20
- data/lib/net/connector/storage.rb +15 -0
- data/lib/net/connector/textfsm.rb +72 -0
- data/lib/net/connector/vendor/cisco_ios/tftp_backup.rb +12 -5
- data/lib/net/connector/vendor/cisco_ios/topology.rb +12 -6
- data/lib/net/connector/vendor/cisco_nxos/tftp_backup.rb +11 -4
- data/lib/net/connector/vendor/h3c/tftp_backup.rb +9 -4
- data/lib/net/connector/vendor/h3c/topology.rb +11 -6
- data/lib/net/connector/vendor/h3c.rb +3 -3
- data/lib/net/connector/vendor/hillstone/tftp_backup.rb +15 -6
- data/lib/net/connector/vendor/hillstone/topology.rb +11 -6
- data/lib/net/connector/vendor/huawei/tftp_backup.rb +9 -3
- data/lib/net/connector/vendor/palo_alto/running_config.rb +1 -1
- data/lib/net/connector/vendor/palo_alto/tftp_backup.rb +17 -8
- data/lib/net/connector/vendor/palo_alto/topology.rb +9 -7
- data/lib/net/connector/vendor/radware/tftp_backup.rb +12 -5
- data/lib/net/connector/vendor/radware/topology.rb +3 -2
- data/lib/net/connector/version.rb +1 -1
- data/lib/net/connector.rb +1 -6
- metadata +54 -38
- data/lib/net/connector/device/interface_description.rb +0 -44
- data/lib/net/connector/device/interface_name.rb +0 -56
- data/lib/net/connector/engine/base.rb +0 -4
- data/lib/net/connector/engine/log_messages.rb +0 -47
- data/lib/net/connector/engine/profile.rb +0 -4
- data/lib/net/connector/engine.rb +0 -4
- data/lib/net/connector/operations/local_backup.rb +0 -49
- data/lib/net/connector/operations/parse_output.rb +0 -60
- data/lib/net/connector/operations/private_file.rb +0 -25
- data/lib/net/connector/operations/running_config/cisco.rb +0 -14
- data/lib/net/connector/operations/running_config/hillstone.rb +0 -12
- data/lib/net/connector/operations/running_config/palo_alto.rb +0 -12
- data/lib/net/connector/operations/running_config/strategy.rb +0 -3
- data/lib/net/connector/operations/running_config.rb +0 -16
- data/lib/net/connector/operations/tftp/cisco_ios.rb +0 -11
- data/lib/net/connector/operations/tftp/cisco_nxos.rb +0 -11
- data/lib/net/connector/operations/tftp/file_upload.rb +0 -42
- data/lib/net/connector/operations/tftp/h3c.rb +0 -11
- data/lib/net/connector/operations/tftp/hillstone.rb +0 -11
- data/lib/net/connector/operations/tftp/huawei.rb +0 -11
- data/lib/net/connector/operations/tftp/palo_alto.rb +0 -11
- data/lib/net/connector/operations/tftp/radware.rb +0 -11
- data/lib/net/connector/operations/tftp/strategy.rb +0 -75
- data/lib/net/connector/operations/tftp_backup.rb +0 -120
- data/lib/net/connector/operations/topology/cisco.rb +0 -11
- data/lib/net/connector/operations/topology/h3c.rb +0 -11
- data/lib/net/connector/operations/topology/hillstone.rb +0 -11
- data/lib/net/connector/operations/topology/palo_alto.rb +0 -11
- data/lib/net/connector/operations/topology/radware.rb +0 -11
- data/lib/net/connector/operations/topology/strategy.rb +0 -67
- data/lib/net/connector/operations/topology.rb +0 -193
- data/lib/net/connector/operations.rb +0 -20
|
@@ -10,7 +10,13 @@ module Net
|
|
|
10
10
|
class ConfigFile
|
|
11
11
|
FIELDS = {
|
|
12
12
|
"netdisco" => { "url" => ["NETDISCO_URL", :string],
|
|
13
|
-
"page_size" => ["NETDISCO_PAGE_SIZE", :integer]
|
|
13
|
+
"page_size" => ["NETDISCO_PAGE_SIZE", :integer],
|
|
14
|
+
"max_pages" => ["NETDISCO_MAX_PAGES", :integer],
|
|
15
|
+
"max_response_bytes" => ["NETDISCO_MAX_RESPONSE_BYTES", :integer],
|
|
16
|
+
"max_inventory_bytes" => ["NETDISCO_MAX_INVENTORY_BYTES", :integer],
|
|
17
|
+
"max_devices" => ["NETDISCO_MAX_DEVICES", :integer],
|
|
18
|
+
"inventory_timeout" => ["NETDISCO_INVENTORY_TIMEOUT", :number],
|
|
19
|
+
"allow_insecure_http" => ["NETDISCO_ALLOW_INSECURE_HTTP", :boolean] },
|
|
14
20
|
"backup" => { "directory" => ["NET_CONNECTOR_BACKUP_DIRECTORY", :string],
|
|
15
21
|
"concurrency" => ["NET_CONNECTOR_CONCURRENCY", :integer],
|
|
16
22
|
"limit_per_vendor" => ["NET_CONNECTOR_SAMPLE_PER_VENDOR", :integer] },
|
|
@@ -21,12 +27,16 @@ module Net
|
|
|
21
27
|
"host_overrides" => ["NET_CONNECTOR_HOST_OVERRIDES", :json],
|
|
22
28
|
"device_rules" => ["NET_CONNECTOR_DEVICE_RULES", :json] },
|
|
23
29
|
"ssh" => { "protocol" => ["NET_CONNECTOR_PROTOCOL", :string],
|
|
30
|
+
"max_script_output_bytes" => ["NET_CONNECTOR_MAX_SCRIPT_OUTPUT_BYTES", :optional_integer],
|
|
24
31
|
"known_hosts" => ["NET_CONNECTOR_KNOWN_HOSTS", :string],
|
|
25
32
|
"host_key_policy" => ["NET_CONNECTOR_HOST_KEY_POLICY", :string],
|
|
26
33
|
"log_directory" => ["NET_CONNECTOR_LOG_DIRECTORY", :string],
|
|
27
34
|
"log_level" => ["NET_CONNECTOR_LOG_LEVEL", :string] },
|
|
28
35
|
"tftp" => { "server" => ["TFTP_HOST", :string],
|
|
29
|
-
"vrfs" => ["NET_CONNECTOR_TFTP_VRFS", :json]
|
|
36
|
+
"vrfs" => ["NET_CONNECTOR_TFTP_VRFS", :json],
|
|
37
|
+
"h3c_source_file" => ["NET_CONNECTOR_H3C_TFTP_SOURCE_FILE", :string],
|
|
38
|
+
"h3c_wireless_source_file" => ["NET_CONNECTOR_H3C_WIRELESS_TFTP_SOURCE_FILE", :string],
|
|
39
|
+
"huawei_source_file" => ["NET_CONNECTOR_HUAWEI_TFTP_SOURCE_FILE", :string] }
|
|
30
40
|
}.freeze
|
|
31
41
|
|
|
32
42
|
# 安全读取 YAML 配置并映射到环境变量格式。
|
|
@@ -52,7 +62,7 @@ module Net
|
|
|
52
62
|
end
|
|
53
63
|
end
|
|
54
64
|
rescue Psych::Exception => error
|
|
55
|
-
raise ArgumentError, "invalid YAML config (#{error.class})"
|
|
65
|
+
raise ArgumentError, "invalid YAML config (#{error.class})", cause: nil
|
|
56
66
|
end
|
|
57
67
|
|
|
58
68
|
# 按字段类型校验并编码单项配置。
|
|
@@ -62,9 +72,21 @@ module Net
|
|
|
62
72
|
raise ArgumentError, "#{section}.#{name} must be a nonempty String" unless value.is_a?(String) && !value.strip.empty?
|
|
63
73
|
|
|
64
74
|
value
|
|
65
|
-
when :integer
|
|
75
|
+
when :integer, :optional_integer
|
|
76
|
+
return nil if type == :optional_integer && value.nil?
|
|
77
|
+
|
|
66
78
|
raise ArgumentError, "#{section}.#{name} must be a positive Integer" unless value.is_a?(Integer) && value.positive?
|
|
67
79
|
|
|
80
|
+
value.to_s
|
|
81
|
+
when :number
|
|
82
|
+
unless value.is_a?(Numeric) && value.real? && value.finite? && value.positive? && value.to_f.finite?
|
|
83
|
+
raise ArgumentError, "#{section}.#{name} must be a positive finite number"
|
|
84
|
+
end
|
|
85
|
+
|
|
86
|
+
value.to_s
|
|
87
|
+
when :boolean
|
|
88
|
+
raise ArgumentError, "#{section}.#{name} must be true or false" unless [true, false].include?(value)
|
|
89
|
+
|
|
68
90
|
value.to_s
|
|
69
91
|
when :list
|
|
70
92
|
unless value.is_a?(Array) && value.all? { |item| item.is_a?(String) && !item.empty? && !item.include?(",") }
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
3
|
require "ipaddr"
|
|
4
|
-
require_relative "../
|
|
4
|
+
require_relative "../storage/saved_config"
|
|
5
5
|
|
|
6
6
|
module Net
|
|
7
7
|
module Connector
|
|
@@ -44,7 +44,7 @@ module Net
|
|
|
44
44
|
def ready? = issue.nil?
|
|
45
45
|
|
|
46
46
|
# 生成不随名称或厂商信息改变的本地文本文件名。
|
|
47
|
-
def backup_filename =
|
|
47
|
+
def backup_filename = Storage::SavedConfig.filename(host)
|
|
48
48
|
|
|
49
49
|
# 生成设备 CLI 可用的 ASCII 远端文件名。
|
|
50
50
|
def tftp_filename
|
|
@@ -53,13 +53,12 @@ module Net
|
|
|
53
53
|
.gsub(/\A[._-]+|[._-]+\z/, "")
|
|
54
54
|
label = vendor&.to_s || "device" if label.empty?
|
|
55
55
|
strategy = Net::Connector.vendor_class(vendor).profile.tftp_strategy if vendor
|
|
56
|
-
|
|
57
|
-
strategy = Operations::Tftp::Strategy unless strategy.respond_to?(:filename)
|
|
56
|
+
raise UnsupportedOperation, "device has no TFTP strategy" unless strategy
|
|
58
57
|
TftpTarget.validate_path!(strategy.filename(host, label: label))
|
|
59
58
|
end
|
|
60
59
|
|
|
61
60
|
# 按厂商及设备地址实例化连接器。
|
|
62
|
-
def
|
|
61
|
+
def build_connector(**settings)
|
|
63
62
|
raise ArgumentError, "device is not ready: #{issue}" unless ready?
|
|
64
63
|
raise ArgumentError, "host cannot be overridden" if settings.key?(:host)
|
|
65
64
|
|
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require_relative "../engine/error_metadata"
|
|
4
|
+
|
|
5
|
+
module Net
|
|
6
|
+
module Connector
|
|
7
|
+
module Netdisco
|
|
8
|
+
# 报告是新的诊断通道:只接受固定词表,不展开异常消息、调用栈、命令、source 或 line。
|
|
9
|
+
Diagnostic = Data.define(:error_code, :error_type, :phase, :underlying_type,
|
|
10
|
+
:artifact_state, :artifact_phase, :verification)
|
|
11
|
+
|
|
12
|
+
class Diagnostic
|
|
13
|
+
ARTIFACT_STATES = %i[not_committed committed durable reported_uploaded].freeze
|
|
14
|
+
ARTIFACT_PHASES = %i[resolve_parent temporary_file write file_sync rename directory_open directory_sync
|
|
15
|
+
cleanup complete path finalize].freeze
|
|
16
|
+
private_constant :ARTIFACT_STATES, :ARTIFACT_PHASES
|
|
17
|
+
|
|
18
|
+
# 包括手工构造的结果在内,v2 都不能携带自由文本诊断值。
|
|
19
|
+
def initialize(error_code: nil, error_type: nil, phase: nil, underlying_type: nil,
|
|
20
|
+
artifact_state: nil, artifact_phase: nil, verification: nil)
|
|
21
|
+
super(error_code: ErrorMetadata.code(error_code),
|
|
22
|
+
error_type: ErrorMetadata.type(error_type), phase: ErrorMetadata.phase(phase),
|
|
23
|
+
underlying_type: ErrorMetadata.type(underlying_type),
|
|
24
|
+
artifact_state: ARTIFACT_STATES.include?(artifact_state) ? artifact_state : nil,
|
|
25
|
+
artifact_phase: ARTIFACT_PHASES.include?(artifact_phase) ? artifact_phase : nil,
|
|
26
|
+
verification: verification == :device_reported ? verification : nil)
|
|
27
|
+
end
|
|
28
|
+
|
|
29
|
+
# 读取已知错误的白名单属性后立即丢弃异常引用,不把回调对象保留进批次。
|
|
30
|
+
def self.from(error, backup: nil, phase: nil)
|
|
31
|
+
return unless error
|
|
32
|
+
|
|
33
|
+
fields = { error_type: ErrorMetadata.type(error.class.name) || "StandardError", phase: phase }
|
|
34
|
+
if error.is_a?(Net::Connector::Error)
|
|
35
|
+
fields.merge!(error_code: error.code, phase: phase || error.phase)
|
|
36
|
+
fields[:underlying_type] = error.underlying.type if error.underlying.instance_of?(UnderlyingError)
|
|
37
|
+
end
|
|
38
|
+
new(**fields.merge(completion_fields(error, backup, phase)))
|
|
39
|
+
rescue StandardError
|
|
40
|
+
new(error_type: "StandardError", phase: phase)
|
|
41
|
+
end
|
|
42
|
+
|
|
43
|
+
def self.completion_fields(error, backup, phase)
|
|
44
|
+
if error.instance_of?(BackupPersistenceError) && backup.instance_of?(Backup) && error.backup.equal?(backup)
|
|
45
|
+
{ artifact_state: error.receipt.state, artifact_phase: error.receipt.phase,
|
|
46
|
+
underlying_type: error.underlying_type }
|
|
47
|
+
elsif error.instance_of?(TftpCompletionError) && backup.instance_of?(TftpReceipt) && error.receipt.equal?(backup)
|
|
48
|
+
{ artifact_state: :reported_uploaded, artifact_phase: error.code == :transfer_finalize_failed ? :finalize : :path,
|
|
49
|
+
verification: :device_reported, underlying_type: error.underlying_type }
|
|
50
|
+
elsif phase == :report && Storage::PrivateFile.receipt_error?(error)
|
|
51
|
+
{ error_code: error.code, artifact_state: error.receipt.state, artifact_phase: error.receipt.phase,
|
|
52
|
+
underlying_type: error.underlying_type }
|
|
53
|
+
else
|
|
54
|
+
{}
|
|
55
|
+
end
|
|
56
|
+
end
|
|
57
|
+
private_class_method :completion_fields
|
|
58
|
+
end
|
|
59
|
+
end
|
|
60
|
+
end
|
|
61
|
+
end
|
|
@@ -8,101 +8,130 @@ module Net
|
|
|
8
8
|
module Netdisco
|
|
9
9
|
# 基于已验证的清单快照组织有上限的并发备份。
|
|
10
10
|
class Fleet
|
|
11
|
+
DEFAULT_SETTING = Object.new.freeze
|
|
12
|
+
private_constant :DEFAULT_SETTING
|
|
13
|
+
|
|
11
14
|
# 组装清单客户端、规则、凭据与结果存储。
|
|
12
15
|
def initialize(client: nil, settings: Settings.new, rules: nil, credentials: nil, connector_factory: nil,
|
|
13
16
|
result_store: ResultStore::Text.new)
|
|
14
17
|
@settings = settings
|
|
15
|
-
@client = client
|
|
16
|
-
@rules = rules
|
|
17
|
-
@credentials = credentials
|
|
18
|
-
@connector_factory = connector_factory || ->(device, connection_settings) { device.
|
|
18
|
+
@client = client
|
|
19
|
+
@rules = rules
|
|
20
|
+
@credentials = credentials
|
|
21
|
+
@connector_factory = connector_factory || ->(device, connection_settings) { device.build_connector(**connection_settings) }
|
|
19
22
|
@result_store = result_store
|
|
20
|
-
raise ArgumentError, "credentials must respond to call"
|
|
23
|
+
raise ArgumentError, "credentials must respond to call" if @credentials && !@credentials.respond_to?(:call)
|
|
21
24
|
raise ArgumentError, "result_store must respond to write" if result_store && !result_store.respond_to?(:write)
|
|
22
25
|
end
|
|
23
26
|
|
|
24
27
|
# 拉取并标记重复管理地址的设备清单。
|
|
25
28
|
def devices
|
|
26
|
-
|
|
27
|
-
raise Client::Error, "Netdisco returned an invalid device inventory" unless rows.is_a?(Array)
|
|
28
|
-
|
|
29
|
-
devices = rows.map { |row| Device.from_row(row, rules: @rules) }
|
|
30
|
-
duplicates = devices.group_by(&:host).reject { |host, entries| host.nil? || entries.size == 1 }
|
|
31
|
-
devices.map do |device|
|
|
32
|
-
duplicates.key?(device.host) ? device.with(issue: :duplicate_host) : device
|
|
33
|
-
end.freeze
|
|
29
|
+
inventory(@settings.snapshot(mode: :inventory))
|
|
34
30
|
end
|
|
35
31
|
|
|
36
32
|
# 规划本地配置备份的设备集合。
|
|
37
33
|
def plan_backup(limit_per_vendor: nil)
|
|
38
|
-
|
|
34
|
+
build_plan(:backup, limit_per_vendor, @settings.snapshot(mode: :backup))
|
|
39
35
|
end
|
|
40
36
|
|
|
41
37
|
# 规划 TFTP 配置备份的设备集合。
|
|
42
38
|
def plan_tftp_backup(limit_per_vendor: 5)
|
|
43
|
-
|
|
39
|
+
build_plan(:tftp, limit_per_vendor, @settings.snapshot(mode: :tftp))
|
|
44
40
|
end
|
|
45
41
|
|
|
46
42
|
# 先拉取完整清单,再并发采集并保存设备配置。
|
|
47
|
-
def backup_all(directory:
|
|
48
|
-
limit_per_vendor: nil, plan: nil, on_start: nil, on_result: nil, on_change: nil
|
|
43
|
+
def backup_all(directory: DEFAULT_SETTING, concurrency: DEFAULT_SETTING,
|
|
44
|
+
limit_per_vendor: nil, plan: nil, on_start: nil, on_result: nil, on_change: nil,
|
|
45
|
+
success_policy: :strict)
|
|
46
|
+
reporting = Report.validate_policy!(success_policy)
|
|
47
|
+
policy = @settings.snapshot(mode: :backup)
|
|
48
|
+
directory = policy.backup_directory if directory.equal?(DEFAULT_SETTING)
|
|
49
|
+
concurrency = policy.concurrency if concurrency.equal?(DEFAULT_SETTING)
|
|
49
50
|
raise ArgumentError, "directory must be a nonempty String" unless directory.is_a?(String) && !directory.empty?
|
|
51
|
+
Worker.new(concurrency: concurrency)
|
|
50
52
|
Planner.validate_limit!(limit_per_vendor)
|
|
51
53
|
validate_callback!(on_result, :on_result)
|
|
52
54
|
validate_callback!(on_start, :on_start)
|
|
53
55
|
validate_callback!(on_change, :on_change)
|
|
54
56
|
|
|
55
|
-
plan ||=
|
|
57
|
+
plan ||= build_plan(:backup, limit_per_vendor, policy)
|
|
56
58
|
validate_plan!(plan, :backup)
|
|
57
59
|
callbacks = [on_result]
|
|
58
60
|
callbacks << ->(item) { on_change.call(item) if item.backup.is_a?(Backup) && item.backup.changed? } if on_change
|
|
59
61
|
target_directory = File.expand_path(directory)
|
|
60
62
|
run_batch(:backup, plan, concurrency: concurrency, report_directory: directory,
|
|
61
|
-
|
|
62
|
-
|
|
63
|
+
output_directory: target_directory, policy: policy, reporting: reporting,
|
|
64
|
+
on_start: on_start, on_result: callbacks.compact) do |device, log_directory|
|
|
65
|
+
backup_one(device, target_directory, log_directory, policy)
|
|
63
66
|
end
|
|
64
67
|
end
|
|
65
68
|
|
|
66
69
|
# 让设备主动导出配置,在执行前检查同批目标文件名冲突。
|
|
67
|
-
def tftp_backup_all(server:, source_files: {}, concurrency:
|
|
70
|
+
def tftp_backup_all(server:, source_files: {}, concurrency: DEFAULT_SETTING,
|
|
68
71
|
limit_per_vendor: 5, vrfs: {}, on_start: nil, on_result: nil, plan: nil,
|
|
69
|
-
report_directory:
|
|
70
|
-
|
|
72
|
+
report_directory: DEFAULT_SETTING, success_policy: :strict)
|
|
73
|
+
reporting = Report.validate_policy!(success_policy)
|
|
74
|
+
policy = @settings.snapshot(mode: :tftp)
|
|
75
|
+
concurrency = policy.concurrency if concurrency.equal?(DEFAULT_SETTING)
|
|
76
|
+
report_directory = policy.backup_directory if report_directory.equal?(DEFAULT_SETTING)
|
|
77
|
+
Worker.new(concurrency: concurrency)
|
|
78
|
+
server = TftpTarget.new(host: server, path: "preflight.cfg").host
|
|
71
79
|
raise ArgumentError, "source_files must be a Hash" unless source_files.is_a?(Hash)
|
|
72
80
|
source_files.each_value { |source_file| TftpTarget.validate_source_file!(source_file) }
|
|
73
|
-
|
|
81
|
+
source_files = source_files.transform_values { |value| value.dup.freeze }.freeze
|
|
82
|
+
vrfs = Settings.validate_vrfs!(vrfs).transform_values { |value| value.dup.freeze }.freeze
|
|
74
83
|
Planner.validate_limit!(limit_per_vendor)
|
|
75
84
|
validate_callback!(on_result, :on_result)
|
|
76
85
|
validate_callback!(on_start, :on_start)
|
|
77
86
|
|
|
78
|
-
plan ||=
|
|
87
|
+
plan ||= build_plan(:tftp, limit_per_vendor, policy)
|
|
79
88
|
validate_plan!(plan, :tftp)
|
|
80
89
|
run_batch(:tftp, plan, concurrency: concurrency, report_directory: report_directory,
|
|
81
|
-
on_start: on_start, on_result: on_result) do |device, log_directory|
|
|
82
|
-
tftp_backup_one(device, server, source_files, log_directory, vrfs)
|
|
90
|
+
policy: policy, reporting: reporting, on_start: on_start, on_result: on_result) do |device, log_directory|
|
|
91
|
+
tftp_backup_one(device, server, source_files, log_directory, vrfs, policy)
|
|
83
92
|
end
|
|
84
93
|
end
|
|
85
94
|
|
|
86
95
|
private
|
|
87
96
|
|
|
97
|
+
# 策略在清单请求之前固定;注入的清单客户端仍由调用方负责其自身生命周期。
|
|
98
|
+
def inventory(policy)
|
|
99
|
+
rows = (@client || @settings.client(policy: policy)).devices
|
|
100
|
+
raise Client::Error, "Netdisco returned an invalid device inventory" unless rows.is_a?(Array)
|
|
101
|
+
|
|
102
|
+
rules = @rules || policy.rules
|
|
103
|
+
devices = rows.map { |row| Device.from_row(row, rules: rules) }
|
|
104
|
+
duplicates = devices.group_by(&:host).reject { |host, entries| host.nil? || entries.size == 1 }
|
|
105
|
+
devices.map { |device| duplicates.key?(device.host) ? device.with(issue: :duplicate_host) : device }.freeze
|
|
106
|
+
end
|
|
107
|
+
|
|
108
|
+
def build_plan(mode, limit, policy)
|
|
109
|
+
Planner.validate_limit!(limit)
|
|
110
|
+
Planner.new(inventory(policy)).call(mode: mode, limit_per_vendor: limit)
|
|
111
|
+
end
|
|
112
|
+
|
|
88
113
|
# 统一组织设备任务、日志目录与报告;Worker 保证单台设备失败不影响其他设备。
|
|
89
|
-
def run_batch(mode, plan, concurrency:, report_directory:, on_start:, on_result:, output_directory: nil)
|
|
114
|
+
def run_batch(mode, plan, concurrency:, report_directory:, policy:, reporting:, on_start:, on_result:, output_directory: nil)
|
|
90
115
|
worker = Worker.new(concurrency: concurrency)
|
|
91
116
|
started_at = Time.now.utc
|
|
117
|
+
started = monotonic
|
|
92
118
|
outcomes = plan.outcomes.dup
|
|
93
119
|
FileUtils.mkdir_p(output_directory, mode: 0o700) if output_directory && !plan.ready.empty?
|
|
94
|
-
log_directory =
|
|
120
|
+
log_directory = policy.log_directory
|
|
95
121
|
FileUtils.mkdir_p(log_directory, mode: 0o700) if log_directory && !plan.ready.empty?
|
|
96
122
|
callback_errors = worker.run(plan.ready, outcomes: outcomes, on_start: on_start, on_result: on_result,
|
|
97
|
-
on_error: ->(device, error) {
|
|
123
|
+
on_error: ->(device, error) { build_outcome(device, :failed, error: error) }) do |device|
|
|
98
124
|
yield device, log_directory
|
|
99
125
|
end
|
|
100
126
|
batch = Batch.new(mode: mode, outcomes: outcomes.freeze, started_at: started_at,
|
|
101
127
|
finished_at: Time.now.utc, callback_errors: callback_errors,
|
|
102
128
|
report_location: nil, report_error: nil)
|
|
129
|
+
batch = Report.new(batch, policy: reporting, duration_ms: ((monotonic - started) * 1000).round)
|
|
103
130
|
save_report(batch, directory: report_directory)
|
|
104
131
|
end
|
|
105
132
|
|
|
133
|
+
def monotonic = Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
|
134
|
+
|
|
106
135
|
# 校验批量任务回调接口。
|
|
107
136
|
def validate_callback!(callback, name)
|
|
108
137
|
raise ArgumentError, "#{name} must respond to call" if callback && !callback.respond_to?(:call)
|
|
@@ -113,18 +142,12 @@ module Net
|
|
|
113
142
|
return batch unless @result_store
|
|
114
143
|
|
|
115
144
|
batch.with(report_location: @result_store.write(batch, directory: directory))
|
|
145
|
+
rescue Storage::PrivateFile::WriteError => error
|
|
146
|
+
# 报告也可能已经原子替换;保留已知位置,不将持久性失败误写成完全没有产物。
|
|
147
|
+
location = error.receipt.path if Storage::PrivateFile.receipt_error?(error) && error.receipt.committed?
|
|
148
|
+
batch.with_report_error(error, location: location)
|
|
116
149
|
rescue StandardError => error
|
|
117
|
-
batch.
|
|
118
|
-
end
|
|
119
|
-
|
|
120
|
-
# 校验厂商 VRF 映射和名称。
|
|
121
|
-
def validate_vrfs!(vrfs)
|
|
122
|
-
unless vrfs.is_a?(Hash) && vrfs.all? { |vendor, value|
|
|
123
|
-
%i[cisco_nxos hillstone].include?(vendor) && value.is_a?(String) &&
|
|
124
|
-
value.match?(/\A[A-Za-z0-9_][A-Za-z0-9_.-]*\z/)
|
|
125
|
-
}
|
|
126
|
-
raise ArgumentError, "vrfs must map supported vendor names to safe VRF names"
|
|
127
|
-
end
|
|
150
|
+
batch.with_report_error(error)
|
|
128
151
|
end
|
|
129
152
|
|
|
130
153
|
# 确认传入计划属于当前备份模式。
|
|
@@ -135,16 +158,16 @@ module Net
|
|
|
135
158
|
end
|
|
136
159
|
|
|
137
160
|
# 构造包含错误类型的单台设备结果。
|
|
138
|
-
def
|
|
161
|
+
def build_outcome(device, status, backup: nil, error: nil)
|
|
139
162
|
Outcome.new(device: device, status: status, backup: backup,
|
|
140
163
|
error_code: error.respond_to?(:code) ? error.code : nil,
|
|
141
|
-
error_type: error&.class&.name)
|
|
164
|
+
error_type: error&.class&.name, diagnostic: Diagnostic.from(error, backup: backup))
|
|
142
165
|
end
|
|
143
166
|
|
|
144
167
|
# 执行单台设备的 TFTP 导出。
|
|
145
|
-
def tftp_backup_one(device, server, source_files, log_directory, vrfs)
|
|
146
|
-
run_one(device, log_directory, success_status: :reported_uploaded,
|
|
147
|
-
close_error_status: :reported_with_error, backup_class:
|
|
168
|
+
def tftp_backup_one(device, server, source_files, log_directory, vrfs, policy)
|
|
169
|
+
run_one(device, log_directory, policy, success_status: :reported_uploaded,
|
|
170
|
+
close_error_status: :reported_with_error, backup_class: TftpReceipt) do |connector|
|
|
148
171
|
source_file = source_files.fetch(device.vendor, nil)
|
|
149
172
|
transfer_settings = { host: server, path: device.tftp_filename, source_file: source_file }
|
|
150
173
|
transfer_settings[:vrf] = vrfs.fetch(device.vendor) if vrfs.key?(device.vendor)
|
|
@@ -153,46 +176,57 @@ module Net
|
|
|
153
176
|
end
|
|
154
177
|
|
|
155
178
|
# 执行单台设备的本地配置采集。
|
|
156
|
-
def backup_one(device, directory, log_directory)
|
|
179
|
+
def backup_one(device, directory, log_directory, policy)
|
|
157
180
|
destination = File.join(directory, device.backup_filename)
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
if backup.is_a?(Backup) && backup.change == :created && previous_digest
|
|
164
|
-
backup.with(previous_sha256: previous_digest,
|
|
165
|
-
change: backup.sha256 == previous_digest ? :unchanged : :changed)
|
|
166
|
-
else
|
|
167
|
-
backup
|
|
181
|
+
Storage::BackupLock.synchronize(destination, host: device.host) do |path_lock|
|
|
182
|
+
Storage::SafeFile.open(destination, missing: true) { |_file, _stat| nil }
|
|
183
|
+
run_one(device, log_directory, policy, success_status: :backed_up,
|
|
184
|
+
close_error_status: :saved_with_error, backup_class: Backup) do |connector|
|
|
185
|
+
path_lock.with_delegated_lock { connector.backup(path: destination) }
|
|
168
186
|
end
|
|
169
187
|
end
|
|
170
188
|
end
|
|
171
189
|
|
|
172
190
|
# 隔离单台设备的凭据、连接、操作和关闭异常。
|
|
173
|
-
def run_one(device, log_directory, success_status:, close_error_status:, backup_class:)
|
|
174
|
-
connection_settings = @credentials.call(device)
|
|
175
|
-
return
|
|
191
|
+
def run_one(device, log_directory, policy, success_status:, close_error_status:, backup_class:)
|
|
192
|
+
connection_settings = @credentials ? @credentials.call(device) : @settings.device_credentials_for(device)
|
|
193
|
+
return build_outcome(device, :missing_credentials) if connection_settings.nil?
|
|
176
194
|
unless connection_settings.is_a?(Hash) && connection_settings.keys.all?(Symbol) &&
|
|
177
195
|
connection_settings[:username].is_a?(String) && !connection_settings[:username].empty? &&
|
|
178
196
|
!connection_settings.key?(:host)
|
|
179
197
|
raise ArgumentError, "credential resolver must return settings with username and without host"
|
|
180
198
|
end
|
|
181
199
|
|
|
200
|
+
connection_settings = policy.connection_options(device.vendor).merge(connection_settings)
|
|
182
201
|
log_name = device.host.tr(":", "_")
|
|
183
202
|
if log_directory
|
|
184
203
|
connection_settings = connection_settings.merge(log_file: File.join(log_directory, "#{log_name}.log"))
|
|
185
204
|
end
|
|
186
205
|
connector = @connector_factory.call(device, connection_settings)
|
|
206
|
+
backup, failure = perform_one(connector, backup_class) { yield connector }
|
|
207
|
+
return build_outcome(device, success_status, backup: backup) unless failure
|
|
208
|
+
|
|
209
|
+
build_outcome(device, backup ? close_error_status : :failed, backup: backup, error: failure)
|
|
210
|
+
rescue StandardError => error
|
|
211
|
+
build_outcome(device, :failed, error: error)
|
|
212
|
+
end
|
|
213
|
+
|
|
214
|
+
# 仅信任库定义且产物类型匹配的提交后错误;普通第三方异常上的 backup 字段不代表完成。
|
|
215
|
+
def perform_one(connector, backup_class)
|
|
187
216
|
backup = nil
|
|
188
217
|
failure = nil
|
|
189
218
|
begin
|
|
190
|
-
result = yield
|
|
219
|
+
result = yield
|
|
191
220
|
raise TypeError, "backup operation did not return #{backup_class}" unless result.is_a?(backup_class)
|
|
192
221
|
|
|
193
222
|
backup = result
|
|
194
223
|
rescue StandardError => error
|
|
195
224
|
failure = error
|
|
225
|
+
if backup_class == Backup && error.instance_of?(BackupPersistenceError) && error.backup.instance_of?(Backup)
|
|
226
|
+
backup = error.backup
|
|
227
|
+
elsif backup_class == TftpReceipt && error.instance_of?(TftpCompletionError) && error.receipt.instance_of?(TftpReceipt)
|
|
228
|
+
backup = error.receipt
|
|
229
|
+
end
|
|
196
230
|
ensure
|
|
197
231
|
begin
|
|
198
232
|
connector.close
|
|
@@ -200,11 +234,7 @@ module Net
|
|
|
200
234
|
failure ||= error
|
|
201
235
|
end
|
|
202
236
|
end
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
outcome(device, backup ? close_error_status : :failed, backup: backup, error: failure)
|
|
206
|
-
rescue StandardError => error
|
|
207
|
-
outcome(device, :failed, error: error)
|
|
237
|
+
[backup, failure]
|
|
208
238
|
end
|
|
209
239
|
end
|
|
210
240
|
end
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Net
|
|
4
|
+
module Connector
|
|
5
|
+
module Netdisco
|
|
6
|
+
# 每次 devices 调用独占一个预算,认证、分页和兼容查询不得重置已用资源或期限。
|
|
7
|
+
class InventoryBudget
|
|
8
|
+
def initialize(options, clock:)
|
|
9
|
+
@options = options
|
|
10
|
+
@clock = clock
|
|
11
|
+
@deadline = @clock.call + options.fetch(:inventory_timeout)
|
|
12
|
+
@bytes = 0
|
|
13
|
+
@devices = 0
|
|
14
|
+
end
|
|
15
|
+
|
|
16
|
+
def remaining
|
|
17
|
+
seconds = @deadline - @clock.call
|
|
18
|
+
raise Client::InventoryTimeout, cause: nil if seconds <= 0
|
|
19
|
+
|
|
20
|
+
seconds
|
|
21
|
+
end
|
|
22
|
+
|
|
23
|
+
# 在追加正文前计数;不能靠 Content-Length 或 JSON 解析后的大小限制内存。
|
|
24
|
+
def consume_bytes(size, response_bytes:)
|
|
25
|
+
remaining
|
|
26
|
+
check_limit!(:max_response_bytes, response_bytes + size)
|
|
27
|
+
check_limit!(:max_inventory_bytes, @bytes + size)
|
|
28
|
+
@bytes += size
|
|
29
|
+
end
|
|
30
|
+
|
|
31
|
+
# 去重前的记录同样占用清单内存;兼容查询也使用这个累计计数。
|
|
32
|
+
def consume_devices(size)
|
|
33
|
+
remaining
|
|
34
|
+
check_limit!(:max_devices, @devices + size)
|
|
35
|
+
@devices += size
|
|
36
|
+
end
|
|
37
|
+
|
|
38
|
+
private
|
|
39
|
+
|
|
40
|
+
def check_limit!(name, value)
|
|
41
|
+
return if value <= @options.fetch(name)
|
|
42
|
+
|
|
43
|
+
raise Client::Error.new("Netdisco inventory exceeded #{name}", code: name), cause: nil
|
|
44
|
+
end
|
|
45
|
+
end
|
|
46
|
+
private_constant :InventoryBudget
|
|
47
|
+
end
|
|
48
|
+
end
|
|
49
|
+
end
|
|
@@ -0,0 +1,115 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "forwardable"
|
|
4
|
+
require_relative "batch"
|
|
5
|
+
|
|
6
|
+
module Net
|
|
7
|
+
module Connector
|
|
8
|
+
module Netdisco
|
|
9
|
+
# 唯一的批量报告契约,分开呈现成功策略、清单覆盖和受控诊断。
|
|
10
|
+
class Report
|
|
11
|
+
extend Forwardable
|
|
12
|
+
SUCCESS = %i[backed_up reported_uploaded].freeze
|
|
13
|
+
ATTEMPTED = [*SUCCESS, :failed, :saved_with_error, :reported_with_error].freeze
|
|
14
|
+
IGNORABLE = %i[filtered sample_limit].freeze
|
|
15
|
+
private_constant :SUCCESS, :ATTEMPTED, :IGNORABLE
|
|
16
|
+
|
|
17
|
+
attr_reader :batch, :policy, :duration_ms, :report_diagnostic
|
|
18
|
+
def_delegators :@batch, *Batch.members, :success?, :status, :counts
|
|
19
|
+
|
|
20
|
+
def self.validate_policy!(policy)
|
|
21
|
+
raise ArgumentError, "success_policy must be strict or selected" unless %i[strict selected].include?(policy)
|
|
22
|
+
|
|
23
|
+
policy
|
|
24
|
+
end
|
|
25
|
+
|
|
26
|
+
def initialize(batch, policy: :strict, duration_ms: nil, report_diagnostic: nil)
|
|
27
|
+
self.class.validate_policy!(policy)
|
|
28
|
+
raise ArgumentError, "report requires a Batch" unless batch.instance_of?(Batch)
|
|
29
|
+
unless duration_ms.nil? || (duration_ms.is_a?(Numeric) && duration_ms.real? && duration_ms.finite? && duration_ms >= 0)
|
|
30
|
+
raise ArgumentError, "duration_ms must be nonnegative and finite"
|
|
31
|
+
end
|
|
32
|
+
unless report_diagnostic.nil? || report_diagnostic.instance_of?(Diagnostic)
|
|
33
|
+
raise ArgumentError, "report_diagnostic must be a Diagnostic"
|
|
34
|
+
end
|
|
35
|
+
|
|
36
|
+
@batch, @policy, @duration_ms, @report_diagnostic = batch, policy, duration_ms, report_diagnostic
|
|
37
|
+
freeze
|
|
38
|
+
end
|
|
39
|
+
|
|
40
|
+
# 严格完成情况与所选成功策略分别保留,未知状态不能当作可忽略跳过。
|
|
41
|
+
def policy_success?
|
|
42
|
+
return batch.success? if policy == :strict
|
|
43
|
+
return false unless callback_errors.empty? && report_error.nil?
|
|
44
|
+
|
|
45
|
+
outcomes.any? { |item| SUCCESS.include?(item.status) } &&
|
|
46
|
+
outcomes.all? { |item| SUCCESS.include?(item.status) || IGNORABLE.include?(item.status) }
|
|
47
|
+
end
|
|
48
|
+
|
|
49
|
+
def with(**attributes)
|
|
50
|
+
self.class.new(batch.with(**attributes), policy: policy, duration_ms: duration_ms,
|
|
51
|
+
report_diagnostic: report_diagnostic)
|
|
52
|
+
end
|
|
53
|
+
|
|
54
|
+
def with_report_error(error, location: nil)
|
|
55
|
+
diagnostic = Diagnostic.from(error, phase: :report)
|
|
56
|
+
self.class.new(batch.with(report_location: location, report_error: diagnostic.error_type),
|
|
57
|
+
policy: policy, duration_ms: duration_ms, report_diagnostic: diagnostic)
|
|
58
|
+
end
|
|
59
|
+
|
|
60
|
+
# 手工构造的批次同样经过诊断白名单,不序列化任意异常内容。
|
|
61
|
+
def summary
|
|
62
|
+
data = batch_summary
|
|
63
|
+
data.merge(
|
|
64
|
+
schema_version: 2, policy: policy, policy_success: policy_success?, duration_ms: duration_ms,
|
|
65
|
+
coverage: { complete: !outcomes.empty? && outcomes.all? { |item| ATTEMPTED.include?(item.status) },
|
|
66
|
+
attempted: outcomes.count { |item| ATTEMPTED.include?(item.status) },
|
|
67
|
+
skipped: outcomes.count { |item| !ATTEMPTED.include?(item.status) } },
|
|
68
|
+
devices: device_summaries(data.fetch(:devices)),
|
|
69
|
+
callback_errors: callback_errors.map { |entry| { host: entry[:host], error_type: ErrorMetadata.type(entry[:error_type]) } },
|
|
70
|
+
report_location: report_location, report_error: ErrorMetadata.type(report_error),
|
|
71
|
+
report_diagnostic: report_diagnostic&.to_h
|
|
72
|
+
)
|
|
73
|
+
end
|
|
74
|
+
|
|
75
|
+
def inspect = "#<#{self.class} policy=#{policy} status=#{status} policy_success=#{policy_success?}>"
|
|
76
|
+
|
|
77
|
+
private
|
|
78
|
+
|
|
79
|
+
# 生成可写入报告及供命令行展示的结构化摘要。
|
|
80
|
+
def batch_summary
|
|
81
|
+
{
|
|
82
|
+
mode: mode, status: status, started_at: started_at.iso8601, finished_at: finished_at.iso8601,
|
|
83
|
+
total: outcomes.size,
|
|
84
|
+
succeeded: outcomes.count(&:success?),
|
|
85
|
+
partial: outcomes.count(&:partial?),
|
|
86
|
+
failed: outcomes.count { |item| item.status == :failed },
|
|
87
|
+
skipped: outcomes.count(&:skipped?),
|
|
88
|
+
counts: counts,
|
|
89
|
+
callback_errors: callback_errors,
|
|
90
|
+
devices: outcomes.map do |item|
|
|
91
|
+
{ host: item.device.host || item.device.source_ip, name: item.device.name,
|
|
92
|
+
vendor: item.device.vendor,
|
|
93
|
+
status: item.status, path: item.backup&.path,
|
|
94
|
+
bytes: item.backup.is_a?(Backup) ? item.backup.bytes : nil,
|
|
95
|
+
sha256: item.backup.is_a?(Backup) ? item.backup.sha256 : nil,
|
|
96
|
+
change: item.backup.is_a?(Backup) ? item.backup.change : nil,
|
|
97
|
+
previous_sha256: item.backup.is_a?(Backup) ? item.backup.previous_sha256 : nil,
|
|
98
|
+
started_at: item.started_at&.iso8601, finished_at: item.finished_at&.iso8601,
|
|
99
|
+
duration_ms: item.duration_ms,
|
|
100
|
+
error_code: item.error_code, error_type: item.error_type }
|
|
101
|
+
end
|
|
102
|
+
}
|
|
103
|
+
end
|
|
104
|
+
|
|
105
|
+
def device_summaries(entries)
|
|
106
|
+
entries.zip(outcomes).map do |entry, item|
|
|
107
|
+
diagnostic = item.diagnostic || Diagnostic.new(error_code: item.error_code, error_type: item.error_type)
|
|
108
|
+
entry.merge(error_code: diagnostic.error_code, error_type: diagnostic.error_type,
|
|
109
|
+
diagnostic: diagnostic.to_h.except(:error_code, :error_type))
|
|
110
|
+
end
|
|
111
|
+
end
|
|
112
|
+
end
|
|
113
|
+
end
|
|
114
|
+
end
|
|
115
|
+
end
|
|
@@ -3,7 +3,7 @@
|
|
|
3
3
|
require "json"
|
|
4
4
|
require "fileutils"
|
|
5
5
|
require "securerandom"
|
|
6
|
-
require_relative "../
|
|
6
|
+
require_relative "../storage/private_file"
|
|
7
7
|
|
|
8
8
|
module Net
|
|
9
9
|
module Connector
|
|
@@ -17,7 +17,7 @@ module Net
|
|
|
17
17
|
FileUtils.mkdir_p(destination, mode: 0o700)
|
|
18
18
|
filename = "netdisco-#{batch.mode}-#{batch.started_at.strftime("%Y%m%dT%H%M%SZ")}-#{SecureRandom.hex(4)}.json"
|
|
19
19
|
path = File.join(destination, filename)
|
|
20
|
-
|
|
20
|
+
Storage::PrivateFile.write(path, JSON.pretty_generate(batch.summary)).path
|
|
21
21
|
end
|
|
22
22
|
end
|
|
23
23
|
|