net-connector 0.4.0 → 0.4.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +110 -68
- data/CONTRIBUTING.md +40 -0
- data/README.md +140 -95
- data/SECURITY.md +11 -0
- data/docs/VERIFICATION.md +129 -5
- data/docs/architecture.md +321 -248
- data/lib/net/connector/device/base.rb +12 -4
- data/lib/net/connector/device/interface_description.rb +3 -0
- data/lib/net/connector/device/profile/builder.rb +236 -0
- data/lib/net/connector/device/profile.rb +2 -227
- data/lib/net/connector/device/running_config/strategy.rb +5 -0
- data/lib/net/connector/device/running_config.rb +7 -0
- data/lib/net/connector/engine/command.rb +14 -2
- data/lib/net/connector/engine/configuration.rb +10 -7
- data/lib/net/connector/engine/dialogue.rb +53 -28
- data/lib/net/connector/engine/errors.rb +41 -54
- data/lib/net/connector/engine/execution.rb +32 -5
- data/lib/net/connector/engine/log.rb +23 -17
- data/lib/net/connector/engine/session.rb +39 -7
- data/lib/net/connector/engine/terminal_renderer.rb +7 -4
- data/lib/net/connector/netdisco/batch.rb +31 -3
- data/lib/net/connector/netdisco/cli.rb +92 -50
- data/lib/net/connector/netdisco/client.rb +206 -73
- data/lib/net/connector/netdisco/config_file.rb +26 -4
- data/lib/net/connector/netdisco/device.rb +4 -8
- data/lib/net/connector/netdisco/diagnostic.rb +91 -0
- data/lib/net/connector/netdisco/fleet.rb +105 -108
- data/lib/net/connector/netdisco/inventory_budget.rb +49 -0
- data/lib/net/connector/netdisco/plan.rb +77 -0
- data/lib/net/connector/netdisco/planner.rb +12 -23
- data/lib/net/connector/netdisco/report.rb +94 -0
- data/lib/net/connector/netdisco/rules.rb +28 -5
- data/lib/net/connector/netdisco/settings.rb +194 -85
- data/lib/net/connector/netdisco/worker.rb +43 -27
- data/lib/net/connector/operations/backup_lock.rb +116 -0
- data/lib/net/connector/operations/local_backup.rb +49 -9
- data/lib/net/connector/operations/parse_output.rb +20 -3
- data/lib/net/connector/operations/private_file.rb +94 -5
- data/lib/net/connector/operations/safe_file.rb +62 -0
- data/lib/net/connector/operations/saved_config/legacy_index.rb +109 -0
- data/lib/net/connector/operations/saved_config.rb +40 -10
- data/lib/net/connector/operations/tftp/file_upload.rb +55 -0
- data/lib/net/connector/operations/tftp/strategy.rb +9 -11
- data/lib/net/connector/operations/tftp_backup.rb +78 -15
- data/lib/net/connector/operations/tftp_receipt.rb +73 -0
- data/lib/net/connector/operations/topology/immediate_strategy.rb +45 -0
- data/lib/net/connector/operations/topology/strategy.rb +34 -0
- data/lib/net/connector/operations/topology.rb +98 -29
- data/lib/net/connector/operations.rb +6 -0
- data/lib/net/connector/vendor/cisco_ios/running_config.rb +1 -0
- data/lib/net/connector/vendor/cisco_ios/tftp_backup.rb +9 -2
- data/lib/net/connector/vendor/cisco_ios/topology.rb +18 -3
- data/lib/net/connector/vendor/cisco_nxos/running_config.rb +1 -0
- data/lib/net/connector/vendor/cisco_nxos/tftp_backup.rb +8 -1
- data/lib/net/connector/vendor/h3c/tftp_backup.rb +7 -18
- data/lib/net/connector/vendor/h3c/topology.rb +17 -3
- data/lib/net/connector/vendor/hillstone/running_config.rb +1 -0
- data/lib/net/connector/vendor/hillstone/tftp_backup.rb +16 -3
- data/lib/net/connector/vendor/hillstone/topology.rb +17 -3
- data/lib/net/connector/vendor/huawei/tftp_backup.rb +8 -18
- data/lib/net/connector/vendor/palo_alto/running_config.rb +4 -0
- data/lib/net/connector/vendor/palo_alto/tftp_backup.rb +19 -8
- data/lib/net/connector/vendor/palo_alto/topology.rb +16 -1
- data/lib/net/connector/vendor/radware/tftp_backup.rb +11 -4
- data/lib/net/connector/vendor/radware/topology.rb +3 -0
- data/lib/net/connector/version.rb +1 -1
- metadata +60 -28
|
@@ -10,7 +10,13 @@ module Net
|
|
|
10
10
|
class ConfigFile
|
|
11
11
|
FIELDS = {
|
|
12
12
|
"netdisco" => { "url" => ["NETDISCO_URL", :string],
|
|
13
|
-
"page_size" => ["NETDISCO_PAGE_SIZE", :integer]
|
|
13
|
+
"page_size" => ["NETDISCO_PAGE_SIZE", :integer],
|
|
14
|
+
"max_pages" => ["NETDISCO_MAX_PAGES", :integer],
|
|
15
|
+
"max_response_bytes" => ["NETDISCO_MAX_RESPONSE_BYTES", :integer],
|
|
16
|
+
"max_inventory_bytes" => ["NETDISCO_MAX_INVENTORY_BYTES", :integer],
|
|
17
|
+
"max_devices" => ["NETDISCO_MAX_DEVICES", :integer],
|
|
18
|
+
"inventory_timeout" => ["NETDISCO_INVENTORY_TIMEOUT", :number],
|
|
19
|
+
"allow_insecure_http" => ["NETDISCO_ALLOW_INSECURE_HTTP", :boolean] },
|
|
14
20
|
"backup" => { "directory" => ["NET_CONNECTOR_BACKUP_DIRECTORY", :string],
|
|
15
21
|
"concurrency" => ["NET_CONNECTOR_CONCURRENCY", :integer],
|
|
16
22
|
"limit_per_vendor" => ["NET_CONNECTOR_SAMPLE_PER_VENDOR", :integer] },
|
|
@@ -21,12 +27,16 @@ module Net
|
|
|
21
27
|
"host_overrides" => ["NET_CONNECTOR_HOST_OVERRIDES", :json],
|
|
22
28
|
"device_rules" => ["NET_CONNECTOR_DEVICE_RULES", :json] },
|
|
23
29
|
"ssh" => { "protocol" => ["NET_CONNECTOR_PROTOCOL", :string],
|
|
30
|
+
"max_script_output_bytes" => ["NET_CONNECTOR_MAX_SCRIPT_OUTPUT_BYTES", :optional_integer],
|
|
24
31
|
"known_hosts" => ["NET_CONNECTOR_KNOWN_HOSTS", :string],
|
|
25
32
|
"host_key_policy" => ["NET_CONNECTOR_HOST_KEY_POLICY", :string],
|
|
26
33
|
"log_directory" => ["NET_CONNECTOR_LOG_DIRECTORY", :string],
|
|
27
34
|
"log_level" => ["NET_CONNECTOR_LOG_LEVEL", :string] },
|
|
28
35
|
"tftp" => { "server" => ["TFTP_HOST", :string],
|
|
29
|
-
"vrfs" => ["NET_CONNECTOR_TFTP_VRFS", :json]
|
|
36
|
+
"vrfs" => ["NET_CONNECTOR_TFTP_VRFS", :json],
|
|
37
|
+
"h3c_source_file" => ["NET_CONNECTOR_H3C_TFTP_SOURCE_FILE", :string],
|
|
38
|
+
"h3c_wireless_source_file" => ["NET_CONNECTOR_H3C_WIRELESS_TFTP_SOURCE_FILE", :string],
|
|
39
|
+
"huawei_source_file" => ["NET_CONNECTOR_HUAWEI_TFTP_SOURCE_FILE", :string] }
|
|
30
40
|
}.freeze
|
|
31
41
|
|
|
32
42
|
# 安全读取 YAML 配置并映射到环境变量格式。
|
|
@@ -52,7 +62,7 @@ module Net
|
|
|
52
62
|
end
|
|
53
63
|
end
|
|
54
64
|
rescue Psych::Exception => error
|
|
55
|
-
raise ArgumentError, "invalid YAML config (#{error.class})"
|
|
65
|
+
raise ArgumentError, "invalid YAML config (#{error.class})", cause: nil
|
|
56
66
|
end
|
|
57
67
|
|
|
58
68
|
# 按字段类型校验并编码单项配置。
|
|
@@ -62,9 +72,21 @@ module Net
|
|
|
62
72
|
raise ArgumentError, "#{section}.#{name} must be a nonempty String" unless value.is_a?(String) && !value.strip.empty?
|
|
63
73
|
|
|
64
74
|
value
|
|
65
|
-
when :integer
|
|
75
|
+
when :integer, :optional_integer
|
|
76
|
+
return nil if type == :optional_integer && value.nil?
|
|
77
|
+
|
|
66
78
|
raise ArgumentError, "#{section}.#{name} must be a positive Integer" unless value.is_a?(Integer) && value.positive?
|
|
67
79
|
|
|
80
|
+
value.to_s
|
|
81
|
+
when :number
|
|
82
|
+
unless value.is_a?(Numeric) && value.real? && value.finite? && value.positive? && value.to_f.finite?
|
|
83
|
+
raise ArgumentError, "#{section}.#{name} must be a positive finite number"
|
|
84
|
+
end
|
|
85
|
+
|
|
86
|
+
value.to_s
|
|
87
|
+
when :boolean
|
|
88
|
+
raise ArgumentError, "#{section}.#{name} must be true or false" unless [true, false].include?(value)
|
|
89
|
+
|
|
68
90
|
value.to_s
|
|
69
91
|
when :list
|
|
70
92
|
unless value.is_a?(Array) && value.all? { |item| item.is_a?(String) && !item.empty? && !item.include?(",") }
|
|
@@ -48,18 +48,14 @@ module Net
|
|
|
48
48
|
|
|
49
49
|
# 生成设备 CLI 可用的 ASCII 远端文件名。
|
|
50
50
|
def tftp_filename
|
|
51
|
-
return "running-config.xml" if vendor == :palo_alto
|
|
52
|
-
|
|
53
51
|
label = name.to_s.encode(Encoding::US_ASCII, invalid: :replace, undef: :replace, replace: "-")
|
|
54
52
|
.gsub(/[^A-Za-z0-9._-]+/, "-").byteslice(0, 180)
|
|
55
53
|
.gsub(/\A[._-]+|[._-]+\z/, "")
|
|
56
54
|
label = vendor&.to_s || "device" if label.empty?
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
end
|
|
62
|
-
TftpTarget.filename(host, extension: extension, label: label)
|
|
55
|
+
strategy = Net::Connector.vendor_class(vendor).profile.tftp_strategy if vendor
|
|
56
|
+
# 旧版自定义策略只需实现实例接口,缺少命名接口时沿用通用 cfg 名称。
|
|
57
|
+
strategy = Operations::Tftp::Strategy unless strategy.respond_to?(:filename)
|
|
58
|
+
TftpTarget.validate_path!(strategy.filename(host, label: label))
|
|
63
59
|
end
|
|
64
60
|
|
|
65
61
|
# 按厂商及设备地址实例化连接器。
|
|
@@ -0,0 +1,91 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Net
|
|
4
|
+
module Connector
|
|
5
|
+
module Netdisco
|
|
6
|
+
# 报告是新的诊断通道:只接受固定词表,不展开异常消息、调用栈、命令、source 或 line。
|
|
7
|
+
Diagnostic = Data.define(:error_code, :error_type, :phase, :underlying_type,
|
|
8
|
+
:artifact_state, :artifact_phase, :verification)
|
|
9
|
+
|
|
10
|
+
class Diagnostic
|
|
11
|
+
CONNECTOR_TYPES = %w[Error ConnectionError AuthenticationError LoginTimeout CommandTimeout WriteTimeout
|
|
12
|
+
PromptError ConnectionClosed TransportError DeviceError ScriptError InternalError
|
|
13
|
+
OutputLimitExceeded ScriptOutputLimitExceeded SessionBusy UnsupportedOperation LogError ParsingError BackupBusy
|
|
14
|
+
BackupPersistenceError TftpCompletionError SavedConfigChanged].freeze
|
|
15
|
+
TYPES = (CONNECTOR_TYPES.map { |name| "Net::Connector::#{name}" } +
|
|
16
|
+
%w[StandardError RuntimeError ArgumentError TypeError IOError EOFError SystemCallError ThreadError
|
|
17
|
+
RangeError KeyError IndexError NoMethodError NameError FrozenError NotImplementedError
|
|
18
|
+
Timeout::Error IO::TimeoutError Errno::EIO Errno::EACCES Errno::EPERM Errno::ENOSPC Errno::EROFS
|
|
19
|
+
Errno::ENOENT Errno::ENOTDIR Errno::EISDIR Errno::ELOOP Errno::EINVAL Errno::ENOSYS
|
|
20
|
+
Errno::ENOTSUP Errno::EOPNOTSUPP Errno::ETIMEDOUT Errno::ECONNREFUSED Errno::ECONNRESET
|
|
21
|
+
Net::Connector::Operations::PrivateFile::WriteError
|
|
22
|
+
Net::Connector::Operations::PrivateFile::PersistenceError
|
|
23
|
+
Net::Connector::Operations::PrivateFile::DirectorySyncUnsupported]).freeze
|
|
24
|
+
CODES = (CONNECTOR_TYPES.map { |name| name.gsub(/([a-z])([A-Z])/, '\1_\2').downcase.to_sym } +
|
|
25
|
+
%i[authentication_failed connection_failed connection_refused no_route connection_reset
|
|
26
|
+
connection_timeout host_key_changed host_key_untrusted rsa_too_small unsupported_cipher
|
|
27
|
+
ambiguous_description ambiguous_neighbor confirmation_required description_stages_unsupported
|
|
28
|
+
description_unconfirmed description_unsupported incomplete_configuration interface_missing
|
|
29
|
+
neighbor_discovery_unsupported parse_failed invalid_output_encoding persistence_unconfirmed stale_plan startup_config_missing
|
|
30
|
+
template_invalid template_missing template_unreadable transfer_failed transfer_finalize_failed
|
|
31
|
+
transfer_path_mismatch transfer_path_unconfirmed transfer_unconfirmed uncommitted_configuration
|
|
32
|
+
unrecognized_output unsupported_configuration_format verification_plan_changed candidate_isolation_unavailable
|
|
33
|
+
backup_durability_unsupported backup_finalize_failed backup_persistence_unconfirmed
|
|
34
|
+
file_finalize_failed file_persistence_unconfirmed file_write_failed directory_sync_unsupported]).freeze
|
|
35
|
+
PHASES = %i[connect login enable command script read write close logging backup collect discover parse
|
|
36
|
+
plan apply verify persist tftp_backup report callback].freeze
|
|
37
|
+
ARTIFACT_STATES = %i[not_committed committed durable reported_uploaded].freeze
|
|
38
|
+
ARTIFACT_PHASES = %i[resolve_parent temporary_file write file_sync rename directory_open directory_sync
|
|
39
|
+
cleanup complete path finalize].freeze
|
|
40
|
+
private_constant :CONNECTOR_TYPES, :TYPES, :CODES, :PHASES, :ARTIFACT_STATES, :ARTIFACT_PHASES
|
|
41
|
+
|
|
42
|
+
# 包括手工构造的结果在内,v2 都不能携带自由文本诊断值。
|
|
43
|
+
def initialize(error_code: nil, error_type: nil, phase: nil, underlying_type: nil,
|
|
44
|
+
artifact_state: nil, artifact_phase: nil, verification: nil)
|
|
45
|
+
super(error_code: CODES.include?(error_code) ? error_code : nil,
|
|
46
|
+
error_type: self.class.type(error_type), phase: PHASES.include?(phase) ? phase : nil,
|
|
47
|
+
underlying_type: self.class.type(underlying_type),
|
|
48
|
+
artifact_state: ARTIFACT_STATES.include?(artifact_state) ? artifact_state : nil,
|
|
49
|
+
artifact_phase: ARTIFACT_PHASES.include?(artifact_phase) ? artifact_phase : nil,
|
|
50
|
+
verification: verification == :device_reported ? verification : nil)
|
|
51
|
+
end
|
|
52
|
+
|
|
53
|
+
def self.type(name)
|
|
54
|
+
return if name.nil?
|
|
55
|
+
|
|
56
|
+
name.instance_of?(String) && TYPES.include?(name) ? name.dup.freeze : "StandardError"
|
|
57
|
+
end
|
|
58
|
+
|
|
59
|
+
# 读取已知错误的白名单属性后立即丢弃异常引用,不把回调对象保留进批次。
|
|
60
|
+
def self.from(error, backup: nil, phase: nil)
|
|
61
|
+
return unless error
|
|
62
|
+
|
|
63
|
+
fields = { error_type: type(error.class.name) || "StandardError", phase: phase }
|
|
64
|
+
if error.is_a?(Net::Connector::Error)
|
|
65
|
+
fields.merge!(error_code: error.code, phase: phase || error.phase)
|
|
66
|
+
fields[:underlying_type] = error.underlying.type if error.underlying.instance_of?(UnderlyingError)
|
|
67
|
+
end
|
|
68
|
+
new(**fields.merge(completion_fields(error, backup, phase)))
|
|
69
|
+
rescue StandardError
|
|
70
|
+
new(error_type: "StandardError", phase: phase)
|
|
71
|
+
end
|
|
72
|
+
|
|
73
|
+
def self.completion_fields(error, backup, phase)
|
|
74
|
+
if error.instance_of?(BackupPersistenceError) && backup.instance_of?(Backup) && error.backup.equal?(backup)
|
|
75
|
+
{ artifact_state: error.receipt.state, artifact_phase: error.receipt.phase,
|
|
76
|
+
underlying_type: error.underlying_type }
|
|
77
|
+
elsif error.instance_of?(TftpCompletionError) && backup.instance_of?(TftpBackup) && error.transfer.equal?(backup)
|
|
78
|
+
{ artifact_state: :reported_uploaded, artifact_phase: error.code == :transfer_finalize_failed ? :finalize : :path,
|
|
79
|
+
verification: :device_reported, underlying_type: error.underlying_type }
|
|
80
|
+
elsif phase == :report && Operations::PrivateFile.receipt_error?(error)
|
|
81
|
+
{ error_code: error.code, artifact_state: error.receipt.state, artifact_phase: error.receipt.phase,
|
|
82
|
+
underlying_type: error.underlying_type }
|
|
83
|
+
else
|
|
84
|
+
{}
|
|
85
|
+
end
|
|
86
|
+
end
|
|
87
|
+
private_class_method :completion_fields
|
|
88
|
+
end
|
|
89
|
+
end
|
|
90
|
+
end
|
|
91
|
+
end
|
|
@@ -8,90 +8,117 @@ module Net
|
|
|
8
8
|
module Netdisco
|
|
9
9
|
# 基于已验证的清单快照组织有上限的并发备份。
|
|
10
10
|
class Fleet
|
|
11
|
+
DEFAULT_SETTING = Object.new.freeze
|
|
12
|
+
private_constant :DEFAULT_SETTING
|
|
13
|
+
|
|
11
14
|
# 组装清单客户端、规则、凭据与结果存储。
|
|
12
15
|
def initialize(client: nil, settings: Settings.new, rules: nil, credentials: nil, connector_factory: nil,
|
|
13
16
|
result_store: ResultStore::Text.new)
|
|
14
17
|
@settings = settings
|
|
15
|
-
@client = client
|
|
16
|
-
@rules = rules
|
|
17
|
-
@credentials = credentials
|
|
18
|
+
@client = client
|
|
19
|
+
@rules = rules
|
|
20
|
+
@credentials = credentials
|
|
18
21
|
@connector_factory = connector_factory || ->(device, connection_settings) { device.connector(**connection_settings) }
|
|
19
22
|
@result_store = result_store
|
|
20
|
-
raise ArgumentError, "credentials must respond to call"
|
|
23
|
+
raise ArgumentError, "credentials must respond to call" if @credentials && !@credentials.respond_to?(:call)
|
|
21
24
|
raise ArgumentError, "result_store must respond to write" if result_store && !result_store.respond_to?(:write)
|
|
22
25
|
end
|
|
23
26
|
|
|
24
27
|
# 拉取并标记重复管理地址的设备清单。
|
|
25
28
|
def devices
|
|
26
|
-
|
|
27
|
-
raise Client::Error, "Netdisco returned an invalid device inventory" unless rows.is_a?(Array)
|
|
28
|
-
|
|
29
|
-
devices = rows.map { |row| Device.from_row(row, rules: @rules) }
|
|
30
|
-
duplicates = devices.group_by(&:host).reject { |host, entries| host.nil? || entries.size == 1 }
|
|
31
|
-
devices.map do |device|
|
|
32
|
-
duplicates.key?(device.host) ? device.with(issue: :duplicate_host) : device
|
|
33
|
-
end.freeze
|
|
29
|
+
inventory(@settings.snapshot(mode: :inventory))
|
|
34
30
|
end
|
|
35
31
|
|
|
36
32
|
# 规划本地配置备份的设备集合。
|
|
37
33
|
def plan_backup(limit_per_vendor: nil)
|
|
38
|
-
|
|
34
|
+
build_plan(:backup, limit_per_vendor, @settings.snapshot(mode: :backup))
|
|
39
35
|
end
|
|
40
36
|
|
|
41
37
|
# 规划 TFTP 配置备份的设备集合。
|
|
42
38
|
def plan_tftp_backup(limit_per_vendor: 5)
|
|
43
|
-
|
|
39
|
+
build_plan(:tftp, limit_per_vendor, @settings.snapshot(mode: :tftp))
|
|
44
40
|
end
|
|
45
41
|
|
|
46
42
|
# 先拉取完整清单,再并发采集并保存设备配置。
|
|
47
|
-
def backup_all(directory:
|
|
48
|
-
limit_per_vendor: nil, plan: nil, on_start: nil, on_result: nil, on_change: nil
|
|
43
|
+
def backup_all(directory: DEFAULT_SETTING, concurrency: DEFAULT_SETTING,
|
|
44
|
+
limit_per_vendor: nil, plan: nil, on_start: nil, on_result: nil, on_change: nil,
|
|
45
|
+
success_policy: :strict, report_schema: nil)
|
|
46
|
+
reporting = Report.options(policy: success_policy, schema: report_schema)
|
|
47
|
+
policy = @settings.snapshot(mode: :backup)
|
|
48
|
+
directory = policy.backup_directory if directory.equal?(DEFAULT_SETTING)
|
|
49
|
+
concurrency = policy.concurrency if concurrency.equal?(DEFAULT_SETTING)
|
|
49
50
|
raise ArgumentError, "directory must be a nonempty String" unless directory.is_a?(String) && !directory.empty?
|
|
51
|
+
Worker.new(concurrency: concurrency)
|
|
50
52
|
Planner.validate_limit!(limit_per_vendor)
|
|
51
53
|
validate_callback!(on_result, :on_result)
|
|
52
54
|
validate_callback!(on_start, :on_start)
|
|
53
55
|
validate_callback!(on_change, :on_change)
|
|
54
56
|
|
|
55
|
-
plan ||=
|
|
57
|
+
plan ||= build_plan(:backup, limit_per_vendor, policy)
|
|
56
58
|
validate_plan!(plan, :backup)
|
|
57
59
|
callbacks = [on_result]
|
|
58
60
|
callbacks << ->(item) { on_change.call(item) if item.backup.is_a?(Backup) && item.backup.changed? } if on_change
|
|
59
61
|
target_directory = File.expand_path(directory)
|
|
62
|
+
saved_config = Operations::SavedConfig.new(directory: target_directory, indexed: true)
|
|
60
63
|
run_batch(:backup, plan, concurrency: concurrency, report_directory: directory,
|
|
61
|
-
|
|
62
|
-
|
|
64
|
+
output_directory: target_directory, policy: policy, reporting: reporting,
|
|
65
|
+
on_start: on_start, on_result: callbacks.compact) do |device, log_directory|
|
|
66
|
+
backup_one(device, target_directory, log_directory, policy, saved_config)
|
|
63
67
|
end
|
|
64
68
|
end
|
|
65
69
|
|
|
66
|
-
#
|
|
67
|
-
def tftp_backup_all(server:, source_files: {}, concurrency:
|
|
70
|
+
# 让设备主动导出配置,在执行前检查同批目标文件名冲突。
|
|
71
|
+
def tftp_backup_all(server:, source_files: {}, concurrency: DEFAULT_SETTING,
|
|
68
72
|
limit_per_vendor: 5, vrfs: {}, on_start: nil, on_result: nil, plan: nil,
|
|
69
|
-
report_directory:
|
|
70
|
-
|
|
73
|
+
report_directory: DEFAULT_SETTING, success_policy: :strict, report_schema: nil)
|
|
74
|
+
reporting = Report.options(policy: success_policy, schema: report_schema)
|
|
75
|
+
policy = @settings.snapshot(mode: :tftp)
|
|
76
|
+
concurrency = policy.concurrency if concurrency.equal?(DEFAULT_SETTING)
|
|
77
|
+
report_directory = policy.backup_directory if report_directory.equal?(DEFAULT_SETTING)
|
|
78
|
+
Worker.new(concurrency: concurrency)
|
|
79
|
+
server = TftpTarget.new(host: server, path: "preflight.cfg").host
|
|
71
80
|
raise ArgumentError, "source_files must be a Hash" unless source_files.is_a?(Hash)
|
|
72
81
|
source_files.each_value { |source_file| TftpTarget.validate_source_file!(source_file) }
|
|
73
|
-
|
|
82
|
+
source_files = source_files.transform_values { |value| value.dup.freeze }.freeze
|
|
83
|
+
vrfs = Settings.validate_vrfs!(vrfs).transform_values { |value| value.dup.freeze }.freeze
|
|
74
84
|
Planner.validate_limit!(limit_per_vendor)
|
|
75
85
|
validate_callback!(on_result, :on_result)
|
|
76
86
|
validate_callback!(on_start, :on_start)
|
|
77
87
|
|
|
78
|
-
plan ||=
|
|
88
|
+
plan ||= build_plan(:tftp, limit_per_vendor, policy)
|
|
79
89
|
validate_plan!(plan, :tftp)
|
|
80
90
|
run_batch(:tftp, plan, concurrency: concurrency, report_directory: report_directory,
|
|
81
|
-
on_start: on_start, on_result: on_result) do |device, log_directory|
|
|
82
|
-
tftp_backup_one(device, server, source_files, log_directory, vrfs)
|
|
91
|
+
policy: policy, reporting: reporting, on_start: on_start, on_result: on_result) do |device, log_directory|
|
|
92
|
+
tftp_backup_one(device, server, source_files, log_directory, vrfs, policy)
|
|
83
93
|
end
|
|
84
94
|
end
|
|
85
95
|
|
|
86
96
|
private
|
|
87
97
|
|
|
98
|
+
# 策略在清单请求之前固定;注入的清单客户端仍由调用方负责其自身生命周期。
|
|
99
|
+
def inventory(policy)
|
|
100
|
+
rows = (@client || @settings.client(policy: policy)).devices
|
|
101
|
+
raise Client::Error, "Netdisco returned an invalid device inventory" unless rows.is_a?(Array)
|
|
102
|
+
|
|
103
|
+
rules = @rules || policy.rules
|
|
104
|
+
devices = rows.map { |row| Device.from_row(row, rules: rules) }
|
|
105
|
+
duplicates = devices.group_by(&:host).reject { |host, entries| host.nil? || entries.size == 1 }
|
|
106
|
+
devices.map { |device| duplicates.key?(device.host) ? device.with(issue: :duplicate_host) : device }.freeze
|
|
107
|
+
end
|
|
108
|
+
|
|
109
|
+
def build_plan(mode, limit, policy)
|
|
110
|
+
Planner.validate_limit!(limit)
|
|
111
|
+
Planner.new(inventory(policy)).call(mode: mode, limit_per_vendor: limit)
|
|
112
|
+
end
|
|
113
|
+
|
|
88
114
|
# 统一组织设备任务、日志目录与报告;Worker 保证单台设备失败不影响其他设备。
|
|
89
|
-
def run_batch(mode, plan, concurrency:, report_directory:, on_start:, on_result:, output_directory: nil)
|
|
115
|
+
def run_batch(mode, plan, concurrency:, report_directory:, policy:, reporting:, on_start:, on_result:, output_directory: nil)
|
|
90
116
|
worker = Worker.new(concurrency: concurrency)
|
|
91
117
|
started_at = Time.now.utc
|
|
118
|
+
started = monotonic
|
|
92
119
|
outcomes = plan.outcomes.dup
|
|
93
120
|
FileUtils.mkdir_p(output_directory, mode: 0o700) if output_directory && !plan.ready.empty?
|
|
94
|
-
log_directory =
|
|
121
|
+
log_directory = policy.log_directory
|
|
95
122
|
FileUtils.mkdir_p(log_directory, mode: 0o700) if log_directory && !plan.ready.empty?
|
|
96
123
|
callback_errors = worker.run(plan.ready, outcomes: outcomes, on_start: on_start, on_result: on_result,
|
|
97
124
|
on_error: ->(device, error) { outcome(device, :failed, error: error) }) do |device|
|
|
@@ -100,9 +127,14 @@ module Net
|
|
|
100
127
|
batch = Batch.new(mode: mode, outcomes: outcomes.freeze, started_at: started_at,
|
|
101
128
|
finished_at: Time.now.utc, callback_errors: callback_errors,
|
|
102
129
|
report_location: nil, report_error: nil)
|
|
130
|
+
if reporting.fetch(:schema) == 2
|
|
131
|
+
batch = Report.new(batch, policy: reporting.fetch(:policy), duration_ms: ((monotonic - started) * 1000).round)
|
|
132
|
+
end
|
|
103
133
|
save_report(batch, directory: report_directory)
|
|
104
134
|
end
|
|
105
135
|
|
|
136
|
+
def monotonic = Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
|
137
|
+
|
|
106
138
|
# 校验批量任务回调接口。
|
|
107
139
|
def validate_callback!(callback, name)
|
|
108
140
|
raise ArgumentError, "#{name} must respond to call" if callback && !callback.respond_to?(:call)
|
|
@@ -113,83 +145,35 @@ module Net
|
|
|
113
145
|
return batch unless @result_store
|
|
114
146
|
|
|
115
147
|
batch.with(report_location: @result_store.write(batch, directory: directory))
|
|
148
|
+
rescue Operations::PrivateFile::WriteError => error
|
|
149
|
+
# 报告也可能已经原子替换;保留已知位置,不将持久性失败误写成完全没有产物。
|
|
150
|
+
location = error.receipt.path if Operations::PrivateFile.receipt_error?(error) && error.receipt.committed?
|
|
151
|
+
return batch.with_report_error(error, location: location) if batch.instance_of?(Report)
|
|
152
|
+
|
|
153
|
+
batch.with(report_location: location, report_error: error.class.name)
|
|
116
154
|
rescue StandardError => error
|
|
117
|
-
batch.
|
|
118
|
-
end
|
|
155
|
+
return batch.with_report_error(error) if batch.instance_of?(Report)
|
|
119
156
|
|
|
120
|
-
|
|
121
|
-
def validate_vrfs!(vrfs)
|
|
122
|
-
unless vrfs.is_a?(Hash) && vrfs.all? { |vendor, value|
|
|
123
|
-
%i[cisco_nxos hillstone].include?(vendor) && value.is_a?(String) &&
|
|
124
|
-
value.match?(/\A[A-Za-z0-9_][A-Za-z0-9_.-]*\z/)
|
|
125
|
-
}
|
|
126
|
-
raise ArgumentError, "vrfs must map supported vendor names to safe VRF names"
|
|
127
|
-
end
|
|
157
|
+
batch.with(report_error: error.class.name)
|
|
128
158
|
end
|
|
129
159
|
|
|
130
160
|
# 确认传入计划属于当前备份模式。
|
|
131
161
|
def validate_plan!(plan, mode)
|
|
132
162
|
raise ArgumentError, "plan must be a #{mode} Netdisco plan" unless plan.is_a?(Plan) && plan.mode == mode
|
|
133
163
|
|
|
134
|
-
|
|
135
|
-
ready = plan.ready
|
|
136
|
-
outcomes = plan.outcomes
|
|
137
|
-
unless inventory.is_a?(Array) && inventory.all?(Device) && ready.is_a?(Array) &&
|
|
138
|
-
outcomes.is_a?(Array) && outcomes.size == inventory.size
|
|
139
|
-
raise ArgumentError, "plan inventory and outcomes must have the same device slots"
|
|
140
|
-
end
|
|
141
|
-
|
|
142
|
-
selected = {}
|
|
143
|
-
previous_index = -1
|
|
144
|
-
ready.each do |task|
|
|
145
|
-
unless task.is_a?(Array) && task.size == 2 && task.first.is_a?(Integer)
|
|
146
|
-
raise ArgumentError, "plan contains an invalid device task"
|
|
147
|
-
end
|
|
148
|
-
|
|
149
|
-
index, device = task
|
|
150
|
-
unless index > previous_index && index < inventory.size && inventory[index].equal?(device) && device.ready?
|
|
151
|
-
raise ArgumentError, "plan tasks must reference ready inventory devices in order"
|
|
152
|
-
end
|
|
153
|
-
|
|
154
|
-
selected[index] = true
|
|
155
|
-
previous_index = index
|
|
156
|
-
end
|
|
157
|
-
if mode == :tftp && ready.count { |_index, device| device.vendor == :palo_alto } > 1
|
|
158
|
-
raise ArgumentError, "plan cannot upload multiple Palo Alto devices to one TFTP filename"
|
|
159
|
-
end
|
|
160
|
-
|
|
161
|
-
inventory.each_with_index do |device, index|
|
|
162
|
-
result = outcomes[index]
|
|
163
|
-
next if selected[index] && result.nil?
|
|
164
|
-
|
|
165
|
-
unless !selected[index] && valid_skipped_outcome?(result, device, mode)
|
|
166
|
-
raise ArgumentError, "plan outcome does not match its inventory slot"
|
|
167
|
-
end
|
|
168
|
-
end
|
|
169
|
-
end
|
|
170
|
-
|
|
171
|
-
# 未执行设备必须保留与清单一致的跳过原因,且不能带有执行产物。
|
|
172
|
-
def valid_skipped_outcome?(result, device, mode)
|
|
173
|
-
return false unless result.is_a?(Outcome) && result.device.equal?(device) && result.backup.nil? &&
|
|
174
|
-
result.error_code.nil? && result.error_type.nil? &&
|
|
175
|
-
result.started_at.nil? && result.finished_at.nil?
|
|
176
|
-
|
|
177
|
-
return result.status == device.issue unless device.ready?
|
|
178
|
-
|
|
179
|
-
result.status == :sample_limit ||
|
|
180
|
-
(mode == :tftp && device.vendor == :palo_alto && result.status == :remote_filename_collision)
|
|
164
|
+
plan.validate!
|
|
181
165
|
end
|
|
182
166
|
|
|
183
167
|
# 构造包含错误类型的单台设备结果。
|
|
184
168
|
def outcome(device, status, backup: nil, error: nil)
|
|
185
169
|
Outcome.new(device: device, status: status, backup: backup,
|
|
186
170
|
error_code: error.respond_to?(:code) ? error.code : nil,
|
|
187
|
-
error_type: error&.class&.name)
|
|
171
|
+
error_type: error&.class&.name, diagnostic: Diagnostic.from(error, backup: backup))
|
|
188
172
|
end
|
|
189
173
|
|
|
190
174
|
# 执行单台设备的 TFTP 导出。
|
|
191
|
-
def tftp_backup_one(device, server, source_files, log_directory, vrfs)
|
|
192
|
-
run_one(device, log_directory, success_status: :reported_uploaded,
|
|
175
|
+
def tftp_backup_one(device, server, source_files, log_directory, vrfs, policy)
|
|
176
|
+
run_one(device, log_directory, policy, success_status: :reported_uploaded,
|
|
193
177
|
close_error_status: :reported_with_error, backup_class: TftpBackup) do |connector|
|
|
194
178
|
source_file = source_files.fetch(device.vendor, nil)
|
|
195
179
|
transfer_settings = { host: server, path: device.tftp_filename, source_file: source_file }
|
|
@@ -199,25 +183,26 @@ module Net
|
|
|
199
183
|
end
|
|
200
184
|
|
|
201
185
|
# 执行单台设备的本地配置采集。
|
|
202
|
-
def backup_one(device, directory, log_directory)
|
|
186
|
+
def backup_one(device, directory, log_directory, policy, saved_config)
|
|
203
187
|
destination = File.join(directory, device.backup_filename)
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
backup.with(previous_sha256: previous_digest,
|
|
211
|
-
change: backup.sha256 == previous_digest ? :unchanged : :changed)
|
|
212
|
-
else
|
|
213
|
-
backup
|
|
188
|
+
Operations::BackupLock.synchronize(destination, host: device.host) do |path_lock|
|
|
189
|
+
previous = saved_config.fingerprint(device.host, required: false)
|
|
190
|
+
previous_digest = previous.sha256 if previous && previous.path != destination
|
|
191
|
+
result = run_one(device, log_directory, policy, success_status: :backed_up,
|
|
192
|
+
close_error_status: :saved_with_error, backup_class: Backup) do |connector|
|
|
193
|
+
path_lock.delegate { connector.backup(path: destination) }
|
|
214
194
|
end
|
|
195
|
+
backup = result.backup
|
|
196
|
+
next result unless backup.is_a?(Backup) && backup.change == :created && previous_digest
|
|
197
|
+
|
|
198
|
+
result.with(backup: backup.with(previous_sha256: previous_digest,
|
|
199
|
+
change: backup.sha256 == previous_digest ? :unchanged : :changed))
|
|
215
200
|
end
|
|
216
201
|
end
|
|
217
202
|
|
|
218
203
|
# 隔离单台设备的凭据、连接、操作和关闭异常。
|
|
219
|
-
def run_one(device, log_directory, success_status:, close_error_status:, backup_class:)
|
|
220
|
-
connection_settings = @credentials.call(device)
|
|
204
|
+
def run_one(device, log_directory, policy, success_status:, close_error_status:, backup_class:)
|
|
205
|
+
connection_settings = @credentials ? @credentials.call(device) : @settings.device_credentials_for(device)
|
|
221
206
|
return outcome(device, :missing_credentials) if connection_settings.nil?
|
|
222
207
|
unless connection_settings.is_a?(Hash) && connection_settings.keys.all?(Symbol) &&
|
|
223
208
|
connection_settings[:username].is_a?(String) && !connection_settings[:username].empty? &&
|
|
@@ -225,20 +210,36 @@ module Net
|
|
|
225
210
|
raise ArgumentError, "credential resolver must return settings with username and without host"
|
|
226
211
|
end
|
|
227
212
|
|
|
213
|
+
connection_settings = policy.connection_options(device.vendor).merge(connection_settings)
|
|
228
214
|
log_name = device.host.tr(":", "_")
|
|
229
215
|
if log_directory
|
|
230
216
|
connection_settings = connection_settings.merge(log_file: File.join(log_directory, "#{log_name}.log"))
|
|
231
217
|
end
|
|
232
218
|
connector = @connector_factory.call(device, connection_settings)
|
|
219
|
+
backup, failure = perform_one(connector, backup_class) { yield connector }
|
|
220
|
+
return outcome(device, success_status, backup: backup) unless failure
|
|
221
|
+
|
|
222
|
+
outcome(device, backup ? close_error_status : :failed, backup: backup, error: failure)
|
|
223
|
+
rescue StandardError => error
|
|
224
|
+
outcome(device, :failed, error: error)
|
|
225
|
+
end
|
|
226
|
+
|
|
227
|
+
# 仅信任库定义且产物类型匹配的提交后错误;普通第三方异常上的 backup 字段不代表完成。
|
|
228
|
+
def perform_one(connector, backup_class)
|
|
233
229
|
backup = nil
|
|
234
230
|
failure = nil
|
|
235
231
|
begin
|
|
236
|
-
result = yield
|
|
232
|
+
result = yield
|
|
237
233
|
raise TypeError, "backup operation did not return #{backup_class}" unless result.is_a?(backup_class)
|
|
238
234
|
|
|
239
235
|
backup = result
|
|
240
236
|
rescue StandardError => error
|
|
241
237
|
failure = error
|
|
238
|
+
if backup_class == Backup && error.instance_of?(BackupPersistenceError) && error.backup.instance_of?(Backup)
|
|
239
|
+
backup = error.backup
|
|
240
|
+
elsif backup_class == TftpBackup && error.instance_of?(TftpCompletionError) && error.transfer.instance_of?(TftpBackup)
|
|
241
|
+
backup = error.transfer
|
|
242
|
+
end
|
|
242
243
|
ensure
|
|
243
244
|
begin
|
|
244
245
|
connector.close
|
|
@@ -246,11 +247,7 @@ module Net
|
|
|
246
247
|
failure ||= error
|
|
247
248
|
end
|
|
248
249
|
end
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
outcome(device, backup ? close_error_status : :failed, backup: backup, error: failure)
|
|
252
|
-
rescue StandardError => error
|
|
253
|
-
outcome(device, :failed, error: error)
|
|
250
|
+
[backup, failure]
|
|
254
251
|
end
|
|
255
252
|
end
|
|
256
253
|
end
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Net
|
|
4
|
+
module Connector
|
|
5
|
+
module Netdisco
|
|
6
|
+
# 每次 devices 调用独占一个预算,认证、分页和兼容查询不得重置已用资源或期限。
|
|
7
|
+
class InventoryBudget
|
|
8
|
+
def initialize(options, clock:)
|
|
9
|
+
@options = options
|
|
10
|
+
@clock = clock
|
|
11
|
+
@deadline = @clock.call + options.fetch(:inventory_timeout)
|
|
12
|
+
@bytes = 0
|
|
13
|
+
@devices = 0
|
|
14
|
+
end
|
|
15
|
+
|
|
16
|
+
def remaining
|
|
17
|
+
seconds = @deadline - @clock.call
|
|
18
|
+
raise Client::InventoryTimeout, cause: nil if seconds <= 0
|
|
19
|
+
|
|
20
|
+
seconds
|
|
21
|
+
end
|
|
22
|
+
|
|
23
|
+
# 在追加正文前计数;不能靠 Content-Length 或 JSON 解析后的大小限制内存。
|
|
24
|
+
def consume_bytes(size, response_bytes:)
|
|
25
|
+
remaining
|
|
26
|
+
check_limit!(:max_response_bytes, response_bytes + size)
|
|
27
|
+
check_limit!(:max_inventory_bytes, @bytes + size)
|
|
28
|
+
@bytes += size
|
|
29
|
+
end
|
|
30
|
+
|
|
31
|
+
# 去重前的记录同样占用清单内存;兼容查询也使用这个累计计数。
|
|
32
|
+
def consume_devices(size)
|
|
33
|
+
remaining
|
|
34
|
+
check_limit!(:max_devices, @devices + size)
|
|
35
|
+
@devices += size
|
|
36
|
+
end
|
|
37
|
+
|
|
38
|
+
private
|
|
39
|
+
|
|
40
|
+
def check_limit!(name, value)
|
|
41
|
+
return if value <= @options.fetch(name)
|
|
42
|
+
|
|
43
|
+
raise Client::Error.new("Netdisco inventory exceeded #{name}", code: name), cause: nil
|
|
44
|
+
end
|
|
45
|
+
end
|
|
46
|
+
private_constant :InventoryBudget
|
|
47
|
+
end
|
|
48
|
+
end
|
|
49
|
+
end
|
|
@@ -0,0 +1,77 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Net
|
|
4
|
+
module Connector
|
|
5
|
+
module Netdisco
|
|
6
|
+
# 固定一份清单上的选择结果与跳过原因,供预览和执行复用。
|
|
7
|
+
Plan = Data.define(:mode, :inventory, :ready, :outcomes) do
|
|
8
|
+
# Data 只冻结对象自身;这里复制并冻结可能由调用方修改的计划容器。
|
|
9
|
+
def initialize(mode:, inventory:, ready:, outcomes:)
|
|
10
|
+
unless inventory.is_a?(Array) && ready.is_a?(Array) && outcomes.is_a?(Array)
|
|
11
|
+
raise ArgumentError, "plan inventory, ready tasks, and outcomes must be Arrays"
|
|
12
|
+
end
|
|
13
|
+
|
|
14
|
+
tasks = ready.map { |task| task.is_a?(Array) ? task.dup.freeze : task }.freeze
|
|
15
|
+
super(mode: mode, inventory: inventory.dup.freeze, ready: tasks, outcomes: outcomes.dup.freeze)
|
|
16
|
+
end
|
|
17
|
+
|
|
18
|
+
# 按清单顺序返回选中的设备。
|
|
19
|
+
def selected = ready.map(&:last)
|
|
20
|
+
|
|
21
|
+
# 在任何设备 I/O 前校验完整快照,外部构造或修改的计划也必须遵守相同约束。
|
|
22
|
+
def validate!
|
|
23
|
+
raise ArgumentError, "plan mode must be backup or tftp" unless %i[backup tftp].include?(mode)
|
|
24
|
+
unless inventory.all?(Device) && outcomes.size == inventory.size
|
|
25
|
+
raise ArgumentError, "plan inventory and outcomes must have the same device slots"
|
|
26
|
+
end
|
|
27
|
+
|
|
28
|
+
selected_indices = {}
|
|
29
|
+
filenames = {}
|
|
30
|
+
previous_index = -1
|
|
31
|
+
ready.each do |task|
|
|
32
|
+
unless task.is_a?(Array) && task.size == 2 && task.first.is_a?(Integer)
|
|
33
|
+
raise ArgumentError, "plan contains an invalid device task"
|
|
34
|
+
end
|
|
35
|
+
|
|
36
|
+
index, device = task
|
|
37
|
+
unless index > previous_index && index < inventory.size && inventory[index].equal?(device) && device.ready?
|
|
38
|
+
raise ArgumentError, "plan tasks must reference ready inventory devices in order"
|
|
39
|
+
end
|
|
40
|
+
if mode == :tftp
|
|
41
|
+
filename = device.tftp_filename
|
|
42
|
+
raise ArgumentError, "plan cannot upload multiple devices to one TFTP filename" if filenames.key?(filename)
|
|
43
|
+
|
|
44
|
+
filenames[filename] = true
|
|
45
|
+
end
|
|
46
|
+
selected_indices[index] = true
|
|
47
|
+
previous_index = index
|
|
48
|
+
end
|
|
49
|
+
|
|
50
|
+
inventory.each_with_index do |device, index|
|
|
51
|
+
result = outcomes[index]
|
|
52
|
+
next if selected_indices[index] && result.nil?
|
|
53
|
+
|
|
54
|
+
unless !selected_indices[index] && valid_skipped_outcome?(result, device, filenames)
|
|
55
|
+
raise ArgumentError, "plan outcome does not match its inventory slot"
|
|
56
|
+
end
|
|
57
|
+
end
|
|
58
|
+
self
|
|
59
|
+
end
|
|
60
|
+
|
|
61
|
+
private
|
|
62
|
+
|
|
63
|
+
# 跳过结果不携带执行产物;文件冲突必须有本批次实际执行的目标作为依据。
|
|
64
|
+
def valid_skipped_outcome?(result, device, filenames)
|
|
65
|
+
return false unless result.is_a?(Outcome) && result.device.equal?(device) && result.backup.nil? &&
|
|
66
|
+
result.error_code.nil? && result.error_type.nil? &&
|
|
67
|
+
result.started_at.nil? && result.finished_at.nil?
|
|
68
|
+
|
|
69
|
+
return result.status == device.issue unless device.ready?
|
|
70
|
+
|
|
71
|
+
result.status == :sample_limit ||
|
|
72
|
+
(mode == :tftp && result.status == :remote_filename_collision && filenames.key?(device.tftp_filename))
|
|
73
|
+
end
|
|
74
|
+
end
|
|
75
|
+
end
|
|
76
|
+
end
|
|
77
|
+
end
|