net-connector 0.4.2 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +23 -1
- data/README.md +92 -20
- data/docs/VERIFICATION.md +17 -7
- data/docs/architecture.md +102 -64
- data/examples/netdisco_database.yml +16 -0
- data/lib/net/connector/device/base.rb +27 -107
- data/lib/net/connector/device/local_backup.rb +96 -0
- data/lib/net/connector/device/profile.rb +8 -3
- data/lib/net/connector/device/running_config/strategy.rb +1 -1
- data/lib/net/connector/device/running_config.rb +44 -5
- data/lib/net/connector/device/save_config.rb +21 -0
- data/lib/net/connector/device/tftp/file_upload.rb +53 -0
- data/lib/net/connector/{operations/tftp_receipt.rb → device/tftp/receipt.rb} +18 -20
- data/lib/net/connector/device/tftp/strategy.rb +67 -0
- data/lib/net/connector/device/tftp/target.rb +63 -0
- data/lib/net/connector/device/tftp.rb +117 -0
- data/lib/net/connector/device/topology/immediate_strategy.rb +40 -0
- data/lib/net/connector/device/topology/interface_description.rb +46 -0
- data/lib/net/connector/device/topology/interface_name.rb +58 -0
- data/lib/net/connector/device/topology/strategy.rb +75 -0
- data/lib/net/connector/device/topology.rb +277 -0
- data/lib/net/connector/engine/authentication.rb +2 -2
- data/lib/net/connector/engine/configuration.rb +1 -1
- data/lib/net/connector/engine/dialogue.rb +21 -21
- data/lib/net/connector/engine/error_metadata.rb +50 -0
- data/lib/net/connector/engine/errors.rb +15 -1
- data/lib/net/connector/engine/execution.rb +17 -10
- data/lib/net/connector/engine/log/event.rb +66 -0
- data/lib/net/connector/engine/log/formatter.rb +16 -0
- data/lib/net/connector/engine/log/messages.rb +52 -0
- data/lib/net/connector/engine/log/stream.rb +56 -0
- data/lib/net/connector/engine/log.rb +105 -84
- data/lib/net/connector/engine/session.rb +83 -42
- data/lib/net/connector/engine/terminal_text.rb +33 -0
- data/lib/net/connector/engine/transport.rb +2 -2
- data/lib/net/connector/netdisco/batch.rb +9 -45
- data/lib/net/connector/netdisco/cli.rb +17 -10
- data/lib/net/connector/netdisco/client.rb +4 -3
- data/lib/net/connector/netdisco/config_file.rb +6 -3
- data/lib/net/connector/netdisco/database_client.rb +197 -0
- data/lib/net/connector/netdisco/device.rb +4 -5
- data/lib/net/connector/netdisco/diagnostic.rb +9 -39
- data/lib/net/connector/netdisco/fleet.rb +26 -39
- data/lib/net/connector/netdisco/report.rb +37 -16
- data/lib/net/connector/netdisco/result_store.rb +2 -2
- data/lib/net/connector/netdisco/settings.rb +26 -9
- data/lib/net/connector/netdisco.rb +1 -0
- data/lib/net/connector/{operations → storage}/backup_lock.rb +2 -2
- data/lib/net/connector/{operations → storage}/private_file.rb +2 -7
- data/lib/net/connector/{operations → storage}/safe_file.rb +1 -1
- data/lib/net/connector/{operations → storage}/saved_config.rb +10 -35
- data/lib/net/connector/storage.rb +15 -0
- data/lib/net/connector/textfsm.rb +72 -0
- data/lib/net/connector/vendor/cisco_ios/tftp_backup.rb +3 -3
- data/lib/net/connector/vendor/cisco_ios/topology.rb +5 -5
- data/lib/net/connector/vendor/cisco_nxos/tftp_backup.rb +3 -3
- data/lib/net/connector/vendor/h3c/tftp_backup.rb +4 -4
- data/lib/net/connector/vendor/h3c/topology.rb +5 -5
- data/lib/net/connector/vendor/h3c.rb +3 -3
- data/lib/net/connector/vendor/hillstone/tftp_backup.rb +3 -3
- data/lib/net/connector/vendor/hillstone/topology.rb +5 -5
- data/lib/net/connector/vendor/huawei/tftp_backup.rb +3 -3
- data/lib/net/connector/vendor/palo_alto/running_config.rb +1 -1
- data/lib/net/connector/vendor/palo_alto/tftp_backup.rb +3 -3
- data/lib/net/connector/vendor/palo_alto/topology.rb +3 -6
- data/lib/net/connector/vendor/radware/tftp_backup.rb +3 -3
- data/lib/net/connector/vendor/radware/topology.rb +2 -2
- data/lib/net/connector/version.rb +1 -1
- data/lib/net/connector.rb +1 -6
- metadata +47 -39
- data/lib/net/connector/device/interface_description.rb +0 -44
- data/lib/net/connector/device/interface_name.rb +0 -56
- data/lib/net/connector/engine/base.rb +0 -4
- data/lib/net/connector/engine/log_messages.rb +0 -47
- data/lib/net/connector/engine/profile.rb +0 -4
- data/lib/net/connector/engine.rb +0 -4
- data/lib/net/connector/operations/local_backup.rb +0 -89
- data/lib/net/connector/operations/parse_output.rb +0 -77
- data/lib/net/connector/operations/running_config/cisco.rb +0 -14
- data/lib/net/connector/operations/running_config/hillstone.rb +0 -12
- data/lib/net/connector/operations/running_config/palo_alto.rb +0 -12
- data/lib/net/connector/operations/running_config/strategy.rb +0 -3
- data/lib/net/connector/operations/running_config.rb +0 -16
- data/lib/net/connector/operations/saved_config/legacy_index.rb +0 -109
- data/lib/net/connector/operations/tftp/cisco_ios.rb +0 -11
- data/lib/net/connector/operations/tftp/cisco_nxos.rb +0 -11
- data/lib/net/connector/operations/tftp/file_upload.rb +0 -55
- data/lib/net/connector/operations/tftp/h3c.rb +0 -11
- data/lib/net/connector/operations/tftp/hillstone.rb +0 -11
- data/lib/net/connector/operations/tftp/huawei.rb +0 -11
- data/lib/net/connector/operations/tftp/palo_alto.rb +0 -11
- data/lib/net/connector/operations/tftp/radware.rb +0 -11
- data/lib/net/connector/operations/tftp/strategy.rb +0 -75
- data/lib/net/connector/operations/tftp_backup.rb +0 -183
- data/lib/net/connector/operations/topology/cisco.rb +0 -11
- data/lib/net/connector/operations/topology/h3c.rb +0 -11
- data/lib/net/connector/operations/topology/hillstone.rb +0 -11
- data/lib/net/connector/operations/topology/immediate_strategy.rb +0 -45
- data/lib/net/connector/operations/topology/palo_alto.rb +0 -11
- data/lib/net/connector/operations/topology/radware.rb +0 -11
- data/lib/net/connector/operations/topology/strategy.rb +0 -87
- data/lib/net/connector/operations/topology.rb +0 -261
- data/lib/net/connector/operations.rb +0 -26
|
@@ -0,0 +1,197 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "timeout"
|
|
4
|
+
require_relative "client"
|
|
5
|
+
|
|
6
|
+
module Net
|
|
7
|
+
module Connector
|
|
8
|
+
module Netdisco
|
|
9
|
+
# SQL 负责业务选择和列别名;适配器只管理只读查询、清单预算与连接生命周期。
|
|
10
|
+
class DatabaseClient
|
|
11
|
+
CONNECTION_ENV = {
|
|
12
|
+
host: "NETDISCO_DB_HOST", port: "NETDISCO_DB_PORT", dbname: "NETDISCO_DB_NAME",
|
|
13
|
+
user: "NETDISCO_DB_USER", password: "NETDISCO_DB_PASS",
|
|
14
|
+
sslmode: "NETDISCO_DB_SSLMODE", sslrootcert: "NETDISCO_DB_SSLROOTCERT",
|
|
15
|
+
connect_timeout: "NETDISCO_DB_CONNECT_TIMEOUT"
|
|
16
|
+
}.freeze
|
|
17
|
+
REQUIRED_CONNECTION_KEYS = %i[host dbname user password].freeze
|
|
18
|
+
DEFAULTS = Client::DEFAULTS.except(:allow_insecure_http).freeze
|
|
19
|
+
|
|
20
|
+
def self.options(query:, query_params: [], **values)
|
|
21
|
+
unless query.is_a?(String) && !query.strip.empty? && !query.include?("\0")
|
|
22
|
+
raise ArgumentError, "query must be a nonempty SQL string without NUL"
|
|
23
|
+
end
|
|
24
|
+
unless query_params.is_a?(Array) && query_params.all? { |value|
|
|
25
|
+
value.nil? || value == true || value == false || value.is_a?(Integer) ||
|
|
26
|
+
(value.is_a?(Float) && value.finite?) || (value.is_a?(String) && !value.include?("\0"))
|
|
27
|
+
}
|
|
28
|
+
raise ArgumentError, "query_params must be an array of JSON scalar values without NUL"
|
|
29
|
+
end
|
|
30
|
+
raise ArgumentError, "unknown database inventory options" unless (values.keys - DEFAULTS.keys).empty?
|
|
31
|
+
|
|
32
|
+
Client.options(**values).slice(*DEFAULTS.keys).merge(
|
|
33
|
+
query: query.dup.freeze,
|
|
34
|
+
query_params: query_params.map { |value| value.is_a?(String) ? value.dup.freeze : value }.freeze
|
|
35
|
+
).freeze
|
|
36
|
+
end
|
|
37
|
+
|
|
38
|
+
def initialize(connection_options:, **options)
|
|
39
|
+
@options = self.class.options(**options)
|
|
40
|
+
@connection_options = validate_connection_options(connection_options)
|
|
41
|
+
end
|
|
42
|
+
|
|
43
|
+
# 每次查询独占连接和游标;只有全部结果验证成功后才把清单交给 Fleet。
|
|
44
|
+
def devices
|
|
45
|
+
require "pg"
|
|
46
|
+
budget = InventoryBudget.new(@options, clock: method(:monotonic))
|
|
47
|
+
connection = nil
|
|
48
|
+
connected = false
|
|
49
|
+
begin
|
|
50
|
+
Timeout.timeout(budget.remaining, Client::InventoryTimeout) do
|
|
51
|
+
# 从握手开始持有连接;PG.connect 尚未返回时被超时中断,调用方无法负责关闭它。
|
|
52
|
+
connection = PG::Connection.connect_start(@connection_options)
|
|
53
|
+
# PostgreSQL NOTICE 可能包含 SQL 或数据,不交给 libpq 默认的 stderr 输出器。
|
|
54
|
+
connection.set_notice_processor { |_notice| nil }
|
|
55
|
+
wait_for_connection(connection, budget)
|
|
56
|
+
connected = true
|
|
57
|
+
# 使用 pg 可中断的 Ruby 等待接口;底层 socket 仍以非阻塞方式工作。
|
|
58
|
+
connection.setnonblocking(false)
|
|
59
|
+
connection.set_client_encoding("UTF8")
|
|
60
|
+
connection.exec("BEGIN READ ONLY").clear
|
|
61
|
+
set_statement_timeout(connection, budget)
|
|
62
|
+
# 即使没有参数也走扩展查询协议,由 PostgreSQL 拒绝多语句和非查询命令。
|
|
63
|
+
connection.exec_params("DECLARE net_connector_inventory NO SCROLL CURSOR FOR\n#{@options.fetch(:query)}",
|
|
64
|
+
@options.fetch(:query_params)).clear
|
|
65
|
+
rows = inventory(connection, budget)
|
|
66
|
+
connection.exec("ROLLBACK").clear
|
|
67
|
+
budget.remaining
|
|
68
|
+
rows
|
|
69
|
+
end
|
|
70
|
+
rescue Client::Error => error
|
|
71
|
+
raise error, cause: nil
|
|
72
|
+
rescue PG::QueryCanceled
|
|
73
|
+
raise Client::InventoryTimeout, cause: nil
|
|
74
|
+
rescue PG::Error, IOError, SystemCallError
|
|
75
|
+
code = connected ? :query_failed : :connection_failed
|
|
76
|
+
raise Client::Error.new("Netdisco database #{code.to_s.tr("_", " ")}", code: code), cause: nil
|
|
77
|
+
ensure
|
|
78
|
+
connection&.finish
|
|
79
|
+
end
|
|
80
|
+
end
|
|
81
|
+
|
|
82
|
+
def inspect = "#<#{self.class}>"
|
|
83
|
+
|
|
84
|
+
private
|
|
85
|
+
|
|
86
|
+
def wait_for_connection(connection, budget)
|
|
87
|
+
deadline = monotonic + @connection_options.fetch(:connect_timeout, budget.remaining).to_f
|
|
88
|
+
loop do
|
|
89
|
+
status = connection.connect_poll
|
|
90
|
+
return if status == PG::PGRES_POLLING_OK
|
|
91
|
+
raise PG::ConnectionBad unless [PG::PGRES_POLLING_READING, PG::PGRES_POLLING_WRITING].include?(status)
|
|
92
|
+
|
|
93
|
+
remaining = [budget.remaining, deadline - monotonic].min
|
|
94
|
+
raise PG::ConnectionBad if remaining <= 0
|
|
95
|
+
|
|
96
|
+
socket = connection.socket_io
|
|
97
|
+
readers = status == PG::PGRES_POLLING_READING ? [socket] : nil
|
|
98
|
+
writers = status == PG::PGRES_POLLING_WRITING ? [socket] : nil
|
|
99
|
+
next if IO.select(readers, writers, [socket], remaining)
|
|
100
|
+
|
|
101
|
+
budget.remaining
|
|
102
|
+
raise PG::ConnectionBad
|
|
103
|
+
end
|
|
104
|
+
end
|
|
105
|
+
|
|
106
|
+
def validate_connection_options(options)
|
|
107
|
+
unless options.is_a?(Hash) && (options.keys - CONNECTION_ENV.keys).empty?
|
|
108
|
+
raise ArgumentError, "connection_options must contain supported PostgreSQL connection keys"
|
|
109
|
+
end
|
|
110
|
+
REQUIRED_CONNECTION_KEYS.each do |key|
|
|
111
|
+
value = options[key]
|
|
112
|
+
unless value.is_a?(String) && !value.strip.empty?
|
|
113
|
+
raise ArgumentError, "#{CONNECTION_ENV.fetch(key)} is required"
|
|
114
|
+
end
|
|
115
|
+
end
|
|
116
|
+
options.each do |key, value|
|
|
117
|
+
valid = value.is_a?(String) && !value.empty? && !value.include?("\0")
|
|
118
|
+
valid ||= %i[port connect_timeout].include?(key) && value.is_a?(Integer)
|
|
119
|
+
raise ArgumentError, "#{CONNECTION_ENV.fetch(key)} is invalid" unless valid
|
|
120
|
+
end
|
|
121
|
+
%i[port connect_timeout].each do |key|
|
|
122
|
+
next unless options.key?(key)
|
|
123
|
+
|
|
124
|
+
value = options.fetch(key).to_s
|
|
125
|
+
maximum = key == :port ? 65_535 : 2_147_483_647
|
|
126
|
+
unless value.match?(/\A\d+\z/) && value.to_i.between?(1, maximum)
|
|
127
|
+
raise ArgumentError, "#{CONNECTION_ENV.fetch(key)} must be a positive integer within range"
|
|
128
|
+
end
|
|
129
|
+
end
|
|
130
|
+
if options[:sslmode] && !%w[disable allow prefer require verify-ca verify-full].include?(options[:sslmode])
|
|
131
|
+
raise ArgumentError, "NETDISCO_DB_SSLMODE is invalid"
|
|
132
|
+
end
|
|
133
|
+
# libpq 会把 dbname 中的 URI/conninfo 再解释为连接参数,不能借此绕过凭据来源。
|
|
134
|
+
if options.fetch(:dbname).match?(/=|\Apostgres(?:ql)?:\/\//)
|
|
135
|
+
raise ArgumentError, "NETDISCO_DB_NAME must be a database name, not a connection string"
|
|
136
|
+
end
|
|
137
|
+
options.transform_values { |value| value.is_a?(String) ? value.dup.freeze : value }.freeze
|
|
138
|
+
end
|
|
139
|
+
|
|
140
|
+
def set_statement_timeout(connection, budget)
|
|
141
|
+
milliseconds = [(budget.remaining * 1000).ceil, 2_147_483_647].min
|
|
142
|
+
connection.exec_params("SELECT set_config('statement_timeout', $1, true)", [milliseconds.to_s]).clear
|
|
143
|
+
end
|
|
144
|
+
|
|
145
|
+
def inventory(connection, budget)
|
|
146
|
+
rows = []
|
|
147
|
+
@options.fetch(:max_pages).times do
|
|
148
|
+
set_statement_timeout(connection, budget)
|
|
149
|
+
count = fetch_page(connection, rows, budget)
|
|
150
|
+
return rows if count < @options.fetch(:page_size)
|
|
151
|
+
end
|
|
152
|
+
raise Client::Error.new("Netdisco inventory exceeded max_pages", code: :max_pages), cause: nil
|
|
153
|
+
end
|
|
154
|
+
|
|
155
|
+
# 单行模式避免 libpq 先缓存整页;Ruby 只积累已经通过预算检查的白名单字段。
|
|
156
|
+
def fetch_page(connection, rows, budget)
|
|
157
|
+
connection.send_query_params("FETCH FORWARD #{@options.fetch(:page_size)} FROM net_connector_inventory", [])
|
|
158
|
+
connection.set_single_row_mode
|
|
159
|
+
count = 0
|
|
160
|
+
response_bytes = 0
|
|
161
|
+
while (result = connection.get_result)
|
|
162
|
+
begin
|
|
163
|
+
result.check
|
|
164
|
+
response_bytes = append_rows(result, rows, budget, response_bytes)
|
|
165
|
+
count += result.ntuples
|
|
166
|
+
ensure
|
|
167
|
+
result.clear
|
|
168
|
+
end
|
|
169
|
+
end
|
|
170
|
+
count
|
|
171
|
+
end
|
|
172
|
+
|
|
173
|
+
def append_rows(result, rows, budget, response_bytes)
|
|
174
|
+
fields = result.fields
|
|
175
|
+
unless fields.include?("ip") && fields.uniq == fields
|
|
176
|
+
raise Client::Error.new("Netdisco database query must return unique column names including ip",
|
|
177
|
+
code: :invalid_inventory), cause: nil
|
|
178
|
+
end
|
|
179
|
+
budget.consume_devices(result.ntuples)
|
|
180
|
+
result.each do |row|
|
|
181
|
+
# 未选用的列也占传输预算,但不会进入清单、计划或报告。
|
|
182
|
+
size = row.sum { |key, value| key.bytesize + (value&.bytesize || 0) }
|
|
183
|
+
budget.consume_bytes(size, response_bytes: response_bytes)
|
|
184
|
+
response_bytes += size
|
|
185
|
+
unless row["ip"].is_a?(String) && !row["ip"].strip.empty?
|
|
186
|
+
raise Client::Error.new("Netdisco returned an invalid device inventory", code: :invalid_inventory), cause: nil
|
|
187
|
+
end
|
|
188
|
+
rows << row.slice(*Client::FIELDS)
|
|
189
|
+
end
|
|
190
|
+
response_bytes
|
|
191
|
+
end
|
|
192
|
+
|
|
193
|
+
def monotonic = Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
|
194
|
+
end
|
|
195
|
+
end
|
|
196
|
+
end
|
|
197
|
+
end
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
3
|
require "ipaddr"
|
|
4
|
-
require_relative "../
|
|
4
|
+
require_relative "../storage/saved_config"
|
|
5
5
|
|
|
6
6
|
module Net
|
|
7
7
|
module Connector
|
|
@@ -44,7 +44,7 @@ module Net
|
|
|
44
44
|
def ready? = issue.nil?
|
|
45
45
|
|
|
46
46
|
# 生成不随名称或厂商信息改变的本地文本文件名。
|
|
47
|
-
def backup_filename =
|
|
47
|
+
def backup_filename = Storage::SavedConfig.filename(host)
|
|
48
48
|
|
|
49
49
|
# 生成设备 CLI 可用的 ASCII 远端文件名。
|
|
50
50
|
def tftp_filename
|
|
@@ -53,13 +53,12 @@ module Net
|
|
|
53
53
|
.gsub(/\A[._-]+|[._-]+\z/, "")
|
|
54
54
|
label = vendor&.to_s || "device" if label.empty?
|
|
55
55
|
strategy = Net::Connector.vendor_class(vendor).profile.tftp_strategy if vendor
|
|
56
|
-
|
|
57
|
-
strategy = Operations::Tftp::Strategy unless strategy.respond_to?(:filename)
|
|
56
|
+
raise UnsupportedOperation, "device has no TFTP strategy" unless strategy
|
|
58
57
|
TftpTarget.validate_path!(strategy.filename(host, label: label))
|
|
59
58
|
end
|
|
60
59
|
|
|
61
60
|
# 按厂商及设备地址实例化连接器。
|
|
62
|
-
def
|
|
61
|
+
def build_connector(**settings)
|
|
63
62
|
raise ArgumentError, "device is not ready: #{issue}" unless ready?
|
|
64
63
|
raise ArgumentError, "host cannot be overridden" if settings.key?(:host)
|
|
65
64
|
|
|
@@ -1,5 +1,7 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
+
require_relative "../engine/error_metadata"
|
|
4
|
+
|
|
3
5
|
module Net
|
|
4
6
|
module Connector
|
|
5
7
|
module Netdisco
|
|
@@ -8,59 +10,27 @@ module Net
|
|
|
8
10
|
:artifact_state, :artifact_phase, :verification)
|
|
9
11
|
|
|
10
12
|
class Diagnostic
|
|
11
|
-
CONNECTOR_TYPES = %w[Error ConnectionError AuthenticationError LoginTimeout CommandTimeout WriteTimeout
|
|
12
|
-
PromptError ConnectionClosed TransportError DeviceError ScriptError InternalError
|
|
13
|
-
OutputLimitExceeded ScriptOutputLimitExceeded SessionBusy UnsupportedOperation LogError ParsingError BackupBusy
|
|
14
|
-
BackupPersistenceError TftpCompletionError SavedConfigChanged].freeze
|
|
15
|
-
TYPES = (CONNECTOR_TYPES.map { |name| "Net::Connector::#{name}" } +
|
|
16
|
-
%w[StandardError RuntimeError ArgumentError TypeError IOError EOFError SystemCallError ThreadError
|
|
17
|
-
RangeError KeyError IndexError NoMethodError NameError FrozenError NotImplementedError
|
|
18
|
-
Timeout::Error IO::TimeoutError Errno::EIO Errno::EACCES Errno::EPERM Errno::ENOSPC Errno::EROFS
|
|
19
|
-
Errno::ENOENT Errno::ENOTDIR Errno::EISDIR Errno::ELOOP Errno::EINVAL Errno::ENOSYS
|
|
20
|
-
Errno::ENOTSUP Errno::EOPNOTSUPP Errno::ETIMEDOUT Errno::ECONNREFUSED Errno::ECONNRESET
|
|
21
|
-
Net::Connector::Operations::PrivateFile::WriteError
|
|
22
|
-
Net::Connector::Operations::PrivateFile::PersistenceError
|
|
23
|
-
Net::Connector::Operations::PrivateFile::DirectorySyncUnsupported]).freeze
|
|
24
|
-
CODES = (CONNECTOR_TYPES.map { |name| name.gsub(/([a-z])([A-Z])/, '\1_\2').downcase.to_sym } +
|
|
25
|
-
%i[authentication_failed connection_failed connection_refused no_route connection_reset
|
|
26
|
-
connection_timeout host_key_changed host_key_untrusted rsa_too_small unsupported_cipher
|
|
27
|
-
ambiguous_description ambiguous_neighbor confirmation_required description_stages_unsupported
|
|
28
|
-
description_unconfirmed description_unsupported incomplete_configuration interface_missing
|
|
29
|
-
neighbor_discovery_unsupported parse_failed invalid_output_encoding persistence_unconfirmed stale_plan startup_config_missing
|
|
30
|
-
template_invalid template_missing template_unreadable transfer_failed transfer_finalize_failed
|
|
31
|
-
transfer_path_mismatch transfer_path_unconfirmed transfer_unconfirmed uncommitted_configuration
|
|
32
|
-
unrecognized_output unsupported_configuration_format verification_plan_changed candidate_isolation_unavailable
|
|
33
|
-
backup_durability_unsupported backup_finalize_failed backup_persistence_unconfirmed
|
|
34
|
-
file_finalize_failed file_persistence_unconfirmed file_write_failed directory_sync_unsupported]).freeze
|
|
35
|
-
PHASES = %i[connect login enable command script read write close logging backup collect discover parse
|
|
36
|
-
plan apply verify persist tftp_backup report callback].freeze
|
|
37
13
|
ARTIFACT_STATES = %i[not_committed committed durable reported_uploaded].freeze
|
|
38
14
|
ARTIFACT_PHASES = %i[resolve_parent temporary_file write file_sync rename directory_open directory_sync
|
|
39
15
|
cleanup complete path finalize].freeze
|
|
40
|
-
private_constant :
|
|
16
|
+
private_constant :ARTIFACT_STATES, :ARTIFACT_PHASES
|
|
41
17
|
|
|
42
18
|
# 包括手工构造的结果在内,v2 都不能携带自由文本诊断值。
|
|
43
19
|
def initialize(error_code: nil, error_type: nil, phase: nil, underlying_type: nil,
|
|
44
20
|
artifact_state: nil, artifact_phase: nil, verification: nil)
|
|
45
|
-
super(error_code:
|
|
46
|
-
error_type:
|
|
47
|
-
underlying_type:
|
|
21
|
+
super(error_code: ErrorMetadata.code(error_code),
|
|
22
|
+
error_type: ErrorMetadata.type(error_type), phase: ErrorMetadata.phase(phase),
|
|
23
|
+
underlying_type: ErrorMetadata.type(underlying_type),
|
|
48
24
|
artifact_state: ARTIFACT_STATES.include?(artifact_state) ? artifact_state : nil,
|
|
49
25
|
artifact_phase: ARTIFACT_PHASES.include?(artifact_phase) ? artifact_phase : nil,
|
|
50
26
|
verification: verification == :device_reported ? verification : nil)
|
|
51
27
|
end
|
|
52
28
|
|
|
53
|
-
def self.type(name)
|
|
54
|
-
return if name.nil?
|
|
55
|
-
|
|
56
|
-
name.instance_of?(String) && TYPES.include?(name) ? name.dup.freeze : "StandardError"
|
|
57
|
-
end
|
|
58
|
-
|
|
59
29
|
# 读取已知错误的白名单属性后立即丢弃异常引用,不把回调对象保留进批次。
|
|
60
30
|
def self.from(error, backup: nil, phase: nil)
|
|
61
31
|
return unless error
|
|
62
32
|
|
|
63
|
-
fields = { error_type: type(error.class.name) || "StandardError", phase: phase }
|
|
33
|
+
fields = { error_type: ErrorMetadata.type(error.class.name) || "StandardError", phase: phase }
|
|
64
34
|
if error.is_a?(Net::Connector::Error)
|
|
65
35
|
fields.merge!(error_code: error.code, phase: phase || error.phase)
|
|
66
36
|
fields[:underlying_type] = error.underlying.type if error.underlying.instance_of?(UnderlyingError)
|
|
@@ -74,10 +44,10 @@ module Net
|
|
|
74
44
|
if error.instance_of?(BackupPersistenceError) && backup.instance_of?(Backup) && error.backup.equal?(backup)
|
|
75
45
|
{ artifact_state: error.receipt.state, artifact_phase: error.receipt.phase,
|
|
76
46
|
underlying_type: error.underlying_type }
|
|
77
|
-
elsif error.instance_of?(TftpCompletionError) && backup.instance_of?(
|
|
47
|
+
elsif error.instance_of?(TftpCompletionError) && backup.instance_of?(TftpReceipt) && error.receipt.equal?(backup)
|
|
78
48
|
{ artifact_state: :reported_uploaded, artifact_phase: error.code == :transfer_finalize_failed ? :finalize : :path,
|
|
79
49
|
verification: :device_reported, underlying_type: error.underlying_type }
|
|
80
|
-
elsif phase == :report &&
|
|
50
|
+
elsif phase == :report && Storage::PrivateFile.receipt_error?(error)
|
|
81
51
|
{ error_code: error.code, artifact_state: error.receipt.state, artifact_phase: error.receipt.phase,
|
|
82
52
|
underlying_type: error.underlying_type }
|
|
83
53
|
else
|
|
@@ -18,7 +18,7 @@ module Net
|
|
|
18
18
|
@client = client
|
|
19
19
|
@rules = rules
|
|
20
20
|
@credentials = credentials
|
|
21
|
-
@connector_factory = connector_factory || ->(device, connection_settings) { device.
|
|
21
|
+
@connector_factory = connector_factory || ->(device, connection_settings) { device.build_connector(**connection_settings) }
|
|
22
22
|
@result_store = result_store
|
|
23
23
|
raise ArgumentError, "credentials must respond to call" if @credentials && !@credentials.respond_to?(:call)
|
|
24
24
|
raise ArgumentError, "result_store must respond to write" if result_store && !result_store.respond_to?(:write)
|
|
@@ -42,8 +42,8 @@ module Net
|
|
|
42
42
|
# 先拉取完整清单,再并发采集并保存设备配置。
|
|
43
43
|
def backup_all(directory: DEFAULT_SETTING, concurrency: DEFAULT_SETTING,
|
|
44
44
|
limit_per_vendor: nil, plan: nil, on_start: nil, on_result: nil, on_change: nil,
|
|
45
|
-
success_policy: :strict
|
|
46
|
-
reporting = Report.
|
|
45
|
+
success_policy: :strict)
|
|
46
|
+
reporting = Report.validate_policy!(success_policy)
|
|
47
47
|
policy = @settings.snapshot(mode: :backup)
|
|
48
48
|
directory = policy.backup_directory if directory.equal?(DEFAULT_SETTING)
|
|
49
49
|
concurrency = policy.concurrency if concurrency.equal?(DEFAULT_SETTING)
|
|
@@ -59,19 +59,18 @@ module Net
|
|
|
59
59
|
callbacks = [on_result]
|
|
60
60
|
callbacks << ->(item) { on_change.call(item) if item.backup.is_a?(Backup) && item.backup.changed? } if on_change
|
|
61
61
|
target_directory = File.expand_path(directory)
|
|
62
|
-
saved_config = Operations::SavedConfig.new(directory: target_directory, indexed: true)
|
|
63
62
|
run_batch(:backup, plan, concurrency: concurrency, report_directory: directory,
|
|
64
63
|
output_directory: target_directory, policy: policy, reporting: reporting,
|
|
65
64
|
on_start: on_start, on_result: callbacks.compact) do |device, log_directory|
|
|
66
|
-
backup_one(device, target_directory, log_directory, policy
|
|
65
|
+
backup_one(device, target_directory, log_directory, policy)
|
|
67
66
|
end
|
|
68
67
|
end
|
|
69
68
|
|
|
70
69
|
# 让设备主动导出配置,在执行前检查同批目标文件名冲突。
|
|
71
70
|
def tftp_backup_all(server:, source_files: {}, concurrency: DEFAULT_SETTING,
|
|
72
71
|
limit_per_vendor: 5, vrfs: {}, on_start: nil, on_result: nil, plan: nil,
|
|
73
|
-
report_directory: DEFAULT_SETTING, success_policy: :strict
|
|
74
|
-
reporting = Report.
|
|
72
|
+
report_directory: DEFAULT_SETTING, success_policy: :strict)
|
|
73
|
+
reporting = Report.validate_policy!(success_policy)
|
|
75
74
|
policy = @settings.snapshot(mode: :tftp)
|
|
76
75
|
concurrency = policy.concurrency if concurrency.equal?(DEFAULT_SETTING)
|
|
77
76
|
report_directory = policy.backup_directory if report_directory.equal?(DEFAULT_SETTING)
|
|
@@ -121,15 +120,13 @@ module Net
|
|
|
121
120
|
log_directory = policy.log_directory
|
|
122
121
|
FileUtils.mkdir_p(log_directory, mode: 0o700) if log_directory && !plan.ready.empty?
|
|
123
122
|
callback_errors = worker.run(plan.ready, outcomes: outcomes, on_start: on_start, on_result: on_result,
|
|
124
|
-
on_error: ->(device, error) {
|
|
123
|
+
on_error: ->(device, error) { build_outcome(device, :failed, error: error) }) do |device|
|
|
125
124
|
yield device, log_directory
|
|
126
125
|
end
|
|
127
126
|
batch = Batch.new(mode: mode, outcomes: outcomes.freeze, started_at: started_at,
|
|
128
127
|
finished_at: Time.now.utc, callback_errors: callback_errors,
|
|
129
128
|
report_location: nil, report_error: nil)
|
|
130
|
-
|
|
131
|
-
batch = Report.new(batch, policy: reporting.fetch(:policy), duration_ms: ((monotonic - started) * 1000).round)
|
|
132
|
-
end
|
|
129
|
+
batch = Report.new(batch, policy: reporting, duration_ms: ((monotonic - started) * 1000).round)
|
|
133
130
|
save_report(batch, directory: report_directory)
|
|
134
131
|
end
|
|
135
132
|
|
|
@@ -145,16 +142,12 @@ module Net
|
|
|
145
142
|
return batch unless @result_store
|
|
146
143
|
|
|
147
144
|
batch.with(report_location: @result_store.write(batch, directory: directory))
|
|
148
|
-
rescue
|
|
145
|
+
rescue Storage::PrivateFile::WriteError => error
|
|
149
146
|
# 报告也可能已经原子替换;保留已知位置,不将持久性失败误写成完全没有产物。
|
|
150
|
-
location = error.receipt.path if
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
batch.with(report_location: location, report_error: error.class.name)
|
|
147
|
+
location = error.receipt.path if Storage::PrivateFile.receipt_error?(error) && error.receipt.committed?
|
|
148
|
+
batch.with_report_error(error, location: location)
|
|
154
149
|
rescue StandardError => error
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
batch.with(report_error: error.class.name)
|
|
150
|
+
batch.with_report_error(error)
|
|
158
151
|
end
|
|
159
152
|
|
|
160
153
|
# 确认传入计划属于当前备份模式。
|
|
@@ -165,7 +158,7 @@ module Net
|
|
|
165
158
|
end
|
|
166
159
|
|
|
167
160
|
# 构造包含错误类型的单台设备结果。
|
|
168
|
-
def
|
|
161
|
+
def build_outcome(device, status, backup: nil, error: nil)
|
|
169
162
|
Outcome.new(device: device, status: status, backup: backup,
|
|
170
163
|
error_code: error.respond_to?(:code) ? error.code : nil,
|
|
171
164
|
error_type: error&.class&.name, diagnostic: Diagnostic.from(error, backup: backup))
|
|
@@ -174,7 +167,7 @@ module Net
|
|
|
174
167
|
# 执行单台设备的 TFTP 导出。
|
|
175
168
|
def tftp_backup_one(device, server, source_files, log_directory, vrfs, policy)
|
|
176
169
|
run_one(device, log_directory, policy, success_status: :reported_uploaded,
|
|
177
|
-
close_error_status: :reported_with_error, backup_class:
|
|
170
|
+
close_error_status: :reported_with_error, backup_class: TftpReceipt) do |connector|
|
|
178
171
|
source_file = source_files.fetch(device.vendor, nil)
|
|
179
172
|
transfer_settings = { host: server, path: device.tftp_filename, source_file: source_file }
|
|
180
173
|
transfer_settings[:vrf] = vrfs.fetch(device.vendor) if vrfs.key?(device.vendor)
|
|
@@ -183,27 +176,21 @@ module Net
|
|
|
183
176
|
end
|
|
184
177
|
|
|
185
178
|
# 执行单台设备的本地配置采集。
|
|
186
|
-
def backup_one(device, directory, log_directory, policy
|
|
179
|
+
def backup_one(device, directory, log_directory, policy)
|
|
187
180
|
destination = File.join(directory, device.backup_filename)
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
path_lock.delegate { connector.backup(path: destination) }
|
|
181
|
+
Storage::BackupLock.synchronize(destination, host: device.host) do |path_lock|
|
|
182
|
+
Storage::SafeFile.open(destination, missing: true) { |_file, _stat| nil }
|
|
183
|
+
run_one(device, log_directory, policy, success_status: :backed_up,
|
|
184
|
+
close_error_status: :saved_with_error, backup_class: Backup) do |connector|
|
|
185
|
+
path_lock.with_delegated_lock { connector.backup(path: destination) }
|
|
194
186
|
end
|
|
195
|
-
backup = result.backup
|
|
196
|
-
next result unless backup.is_a?(Backup) && backup.change == :created && previous_digest
|
|
197
|
-
|
|
198
|
-
result.with(backup: backup.with(previous_sha256: previous_digest,
|
|
199
|
-
change: backup.sha256 == previous_digest ? :unchanged : :changed))
|
|
200
187
|
end
|
|
201
188
|
end
|
|
202
189
|
|
|
203
190
|
# 隔离单台设备的凭据、连接、操作和关闭异常。
|
|
204
191
|
def run_one(device, log_directory, policy, success_status:, close_error_status:, backup_class:)
|
|
205
192
|
connection_settings = @credentials ? @credentials.call(device) : @settings.device_credentials_for(device)
|
|
206
|
-
return
|
|
193
|
+
return build_outcome(device, :missing_credentials) if connection_settings.nil?
|
|
207
194
|
unless connection_settings.is_a?(Hash) && connection_settings.keys.all?(Symbol) &&
|
|
208
195
|
connection_settings[:username].is_a?(String) && !connection_settings[:username].empty? &&
|
|
209
196
|
!connection_settings.key?(:host)
|
|
@@ -217,11 +204,11 @@ module Net
|
|
|
217
204
|
end
|
|
218
205
|
connector = @connector_factory.call(device, connection_settings)
|
|
219
206
|
backup, failure = perform_one(connector, backup_class) { yield connector }
|
|
220
|
-
return
|
|
207
|
+
return build_outcome(device, success_status, backup: backup) unless failure
|
|
221
208
|
|
|
222
|
-
|
|
209
|
+
build_outcome(device, backup ? close_error_status : :failed, backup: backup, error: failure)
|
|
223
210
|
rescue StandardError => error
|
|
224
|
-
|
|
211
|
+
build_outcome(device, :failed, error: error)
|
|
225
212
|
end
|
|
226
213
|
|
|
227
214
|
# 仅信任库定义且产物类型匹配的提交后错误;普通第三方异常上的 backup 字段不代表完成。
|
|
@@ -237,8 +224,8 @@ module Net
|
|
|
237
224
|
failure = error
|
|
238
225
|
if backup_class == Backup && error.instance_of?(BackupPersistenceError) && error.backup.instance_of?(Backup)
|
|
239
226
|
backup = error.backup
|
|
240
|
-
elsif backup_class ==
|
|
241
|
-
backup = error.
|
|
227
|
+
elsif backup_class == TftpReceipt && error.instance_of?(TftpCompletionError) && error.receipt.instance_of?(TftpReceipt)
|
|
228
|
+
backup = error.receipt
|
|
242
229
|
end
|
|
243
230
|
ensure
|
|
244
231
|
begin
|
|
@@ -6,7 +6,7 @@ require_relative "batch"
|
|
|
6
6
|
module Net
|
|
7
7
|
module Connector
|
|
8
8
|
module Netdisco
|
|
9
|
-
#
|
|
9
|
+
# 唯一的批量报告契约,分开呈现成功策略、清单覆盖和受控诊断。
|
|
10
10
|
class Report
|
|
11
11
|
extend Forwardable
|
|
12
12
|
SUCCESS = %i[backed_up reported_uploaded].freeze
|
|
@@ -17,19 +17,14 @@ module Net
|
|
|
17
17
|
attr_reader :batch, :policy, :duration_ms, :report_diagnostic
|
|
18
18
|
def_delegators :@batch, *Batch.members, :success?, :status, :counts
|
|
19
19
|
|
|
20
|
-
|
|
21
|
-
def self.options(policy: :strict, schema: nil)
|
|
20
|
+
def self.validate_policy!(policy)
|
|
22
21
|
raise ArgumentError, "success_policy must be strict or selected" unless %i[strict selected].include?(policy)
|
|
23
22
|
|
|
24
|
-
|
|
25
|
-
raise ArgumentError, "report_schema must be 1 or 2" unless schema.is_a?(Integer) && [1, 2].include?(schema)
|
|
26
|
-
raise ArgumentError, "selected success policy requires report schema 2" if policy == :selected && schema != 2
|
|
27
|
-
|
|
28
|
-
{ policy: policy, schema: schema }.freeze
|
|
23
|
+
policy
|
|
29
24
|
end
|
|
30
25
|
|
|
31
26
|
def initialize(batch, policy: :strict, duration_ms: nil, report_diagnostic: nil)
|
|
32
|
-
self.class.
|
|
27
|
+
self.class.validate_policy!(policy)
|
|
33
28
|
raise ArgumentError, "report requires a Batch" unless batch.instance_of?(Batch)
|
|
34
29
|
unless duration_ms.nil? || (duration_ms.is_a?(Numeric) && duration_ms.real? && duration_ms.finite? && duration_ms >= 0)
|
|
35
30
|
raise ArgumentError, "duration_ms must be nonnegative and finite"
|
|
@@ -42,7 +37,7 @@ module Net
|
|
|
42
37
|
freeze
|
|
43
38
|
end
|
|
44
39
|
|
|
45
|
-
#
|
|
40
|
+
# 严格完成情况与所选成功策略分别保留,未知状态不能当作可忽略跳过。
|
|
46
41
|
def policy_success?
|
|
47
42
|
return batch.success? if policy == :strict
|
|
48
43
|
return false unless callback_errors.empty? && report_error.nil?
|
|
@@ -62,17 +57,17 @@ module Net
|
|
|
62
57
|
policy: policy, duration_ms: duration_ms, report_diagnostic: diagnostic)
|
|
63
58
|
end
|
|
64
59
|
|
|
65
|
-
#
|
|
60
|
+
# 手工构造的批次同样经过诊断白名单,不序列化任意异常内容。
|
|
66
61
|
def summary
|
|
67
|
-
|
|
68
|
-
|
|
62
|
+
data = batch_summary
|
|
63
|
+
data.merge(
|
|
69
64
|
schema_version: 2, policy: policy, policy_success: policy_success?, duration_ms: duration_ms,
|
|
70
65
|
coverage: { complete: !outcomes.empty? && outcomes.all? { |item| ATTEMPTED.include?(item.status) },
|
|
71
66
|
attempted: outcomes.count { |item| ATTEMPTED.include?(item.status) },
|
|
72
67
|
skipped: outcomes.count { |item| !ATTEMPTED.include?(item.status) } },
|
|
73
|
-
devices: device_summaries(
|
|
74
|
-
callback_errors: callback_errors.map { |entry| { host: entry[:host], error_type:
|
|
75
|
-
report_location: report_location, report_error:
|
|
68
|
+
devices: device_summaries(data.fetch(:devices)),
|
|
69
|
+
callback_errors: callback_errors.map { |entry| { host: entry[:host], error_type: ErrorMetadata.type(entry[:error_type]) } },
|
|
70
|
+
report_location: report_location, report_error: ErrorMetadata.type(report_error),
|
|
76
71
|
report_diagnostic: report_diagnostic&.to_h
|
|
77
72
|
)
|
|
78
73
|
end
|
|
@@ -81,6 +76,32 @@ module Net
|
|
|
81
76
|
|
|
82
77
|
private
|
|
83
78
|
|
|
79
|
+
# 生成可写入报告及供命令行展示的结构化摘要。
|
|
80
|
+
def batch_summary
|
|
81
|
+
{
|
|
82
|
+
mode: mode, status: status, started_at: started_at.iso8601, finished_at: finished_at.iso8601,
|
|
83
|
+
total: outcomes.size,
|
|
84
|
+
succeeded: outcomes.count(&:success?),
|
|
85
|
+
partial: outcomes.count(&:partial?),
|
|
86
|
+
failed: outcomes.count { |item| item.status == :failed },
|
|
87
|
+
skipped: outcomes.count(&:skipped?),
|
|
88
|
+
counts: counts,
|
|
89
|
+
callback_errors: callback_errors,
|
|
90
|
+
devices: outcomes.map do |item|
|
|
91
|
+
{ host: item.device.host || item.device.source_ip, name: item.device.name,
|
|
92
|
+
vendor: item.device.vendor,
|
|
93
|
+
status: item.status, path: item.backup&.path,
|
|
94
|
+
bytes: item.backup.is_a?(Backup) ? item.backup.bytes : nil,
|
|
95
|
+
sha256: item.backup.is_a?(Backup) ? item.backup.sha256 : nil,
|
|
96
|
+
change: item.backup.is_a?(Backup) ? item.backup.change : nil,
|
|
97
|
+
previous_sha256: item.backup.is_a?(Backup) ? item.backup.previous_sha256 : nil,
|
|
98
|
+
started_at: item.started_at&.iso8601, finished_at: item.finished_at&.iso8601,
|
|
99
|
+
duration_ms: item.duration_ms,
|
|
100
|
+
error_code: item.error_code, error_type: item.error_type }
|
|
101
|
+
end
|
|
102
|
+
}
|
|
103
|
+
end
|
|
104
|
+
|
|
84
105
|
def device_summaries(entries)
|
|
85
106
|
entries.zip(outcomes).map do |entry, item|
|
|
86
107
|
diagnostic = item.diagnostic || Diagnostic.new(error_code: item.error_code, error_type: item.error_type)
|
|
@@ -3,7 +3,7 @@
|
|
|
3
3
|
require "json"
|
|
4
4
|
require "fileutils"
|
|
5
5
|
require "securerandom"
|
|
6
|
-
require_relative "../
|
|
6
|
+
require_relative "../storage/private_file"
|
|
7
7
|
|
|
8
8
|
module Net
|
|
9
9
|
module Connector
|
|
@@ -17,7 +17,7 @@ module Net
|
|
|
17
17
|
FileUtils.mkdir_p(destination, mode: 0o700)
|
|
18
18
|
filename = "netdisco-#{batch.mode}-#{batch.started_at.strftime("%Y%m%dT%H%M%SZ")}-#{SecureRandom.hex(4)}.json"
|
|
19
19
|
path = File.join(destination, filename)
|
|
20
|
-
|
|
20
|
+
Storage::PrivateFile.write(path, JSON.pretty_generate(batch.summary)).path
|
|
21
21
|
end
|
|
22
22
|
end
|
|
23
23
|
|