net-connector 0.4.1 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +50 -0
- data/CONTRIBUTING.md +40 -0
- data/README.md +93 -18
- data/SECURITY.md +11 -0
- data/docs/VERIFICATION.md +139 -6
- data/docs/architecture.md +248 -39
- data/lib/net/connector/device/base.rb +28 -101
- data/lib/net/connector/device/local_backup.rb +96 -0
- data/lib/net/connector/device/profile.rb +8 -3
- data/lib/net/connector/device/running_config/strategy.rb +1 -1
- data/lib/net/connector/device/running_config.rb +46 -5
- data/lib/net/connector/device/save_config.rb +21 -0
- data/lib/net/connector/device/tftp/file_upload.rb +53 -0
- data/lib/net/connector/device/tftp/receipt.rb +71 -0
- data/lib/net/connector/device/tftp/strategy.rb +67 -0
- data/lib/net/connector/device/tftp/target.rb +63 -0
- data/lib/net/connector/device/tftp.rb +117 -0
- data/lib/net/connector/device/topology/immediate_strategy.rb +40 -0
- data/lib/net/connector/device/topology/interface_description.rb +46 -0
- data/lib/net/connector/device/topology/interface_name.rb +58 -0
- data/lib/net/connector/device/topology/strategy.rb +75 -0
- data/lib/net/connector/device/topology.rb +277 -0
- data/lib/net/connector/engine/authentication.rb +1 -1
- data/lib/net/connector/engine/command.rb +14 -2
- data/lib/net/connector/engine/configuration.rb +11 -8
- data/lib/net/connector/engine/dialogue.rb +59 -35
- data/lib/net/connector/engine/error_metadata.rb +50 -0
- data/lib/net/connector/engine/errors.rb +55 -56
- data/lib/net/connector/engine/execution.rb +44 -10
- data/lib/net/connector/engine/log/event.rb +66 -0
- data/lib/net/connector/engine/log/formatter.rb +16 -0
- data/lib/net/connector/engine/log/messages.rb +52 -0
- data/lib/net/connector/engine/log/stream.rb +56 -0
- data/lib/net/connector/engine/log.rb +107 -84
- data/lib/net/connector/engine/session.rb +115 -43
- data/lib/net/connector/engine/terminal_renderer.rb +7 -4
- data/lib/net/connector/engine/terminal_text.rb +33 -0
- data/lib/net/connector/engine/transport.rb +2 -2
- data/lib/net/connector/netdisco/batch.rb +25 -33
- data/lib/net/connector/netdisco/cli.rb +71 -31
- data/lib/net/connector/netdisco/client.rb +206 -73
- data/lib/net/connector/netdisco/config_file.rb +26 -4
- data/lib/net/connector/netdisco/device.rb +4 -5
- data/lib/net/connector/netdisco/diagnostic.rb +61 -0
- data/lib/net/connector/netdisco/fleet.rb +96 -66
- data/lib/net/connector/netdisco/inventory_budget.rb +49 -0
- data/lib/net/connector/netdisco/report.rb +115 -0
- data/lib/net/connector/netdisco/result_store.rb +2 -2
- data/lib/net/connector/netdisco/rules.rb +28 -5
- data/lib/net/connector/netdisco/settings.rb +188 -85
- data/lib/net/connector/netdisco/worker.rb +26 -17
- data/lib/net/connector/storage/backup_lock.rb +116 -0
- data/lib/net/connector/storage/private_file.rb +109 -0
- data/lib/net/connector/storage/safe_file.rb +62 -0
- data/lib/net/connector/{operations → storage}/saved_config.rb +25 -20
- data/lib/net/connector/storage.rb +15 -0
- data/lib/net/connector/textfsm.rb +72 -0
- data/lib/net/connector/vendor/cisco_ios/tftp_backup.rb +12 -5
- data/lib/net/connector/vendor/cisco_ios/topology.rb +12 -6
- data/lib/net/connector/vendor/cisco_nxos/tftp_backup.rb +11 -4
- data/lib/net/connector/vendor/h3c/tftp_backup.rb +9 -4
- data/lib/net/connector/vendor/h3c/topology.rb +11 -6
- data/lib/net/connector/vendor/h3c.rb +3 -3
- data/lib/net/connector/vendor/hillstone/tftp_backup.rb +15 -6
- data/lib/net/connector/vendor/hillstone/topology.rb +11 -6
- data/lib/net/connector/vendor/huawei/tftp_backup.rb +9 -3
- data/lib/net/connector/vendor/palo_alto/running_config.rb +1 -1
- data/lib/net/connector/vendor/palo_alto/tftp_backup.rb +17 -8
- data/lib/net/connector/vendor/palo_alto/topology.rb +9 -7
- data/lib/net/connector/vendor/radware/tftp_backup.rb +12 -5
- data/lib/net/connector/vendor/radware/topology.rb +3 -2
- data/lib/net/connector/version.rb +1 -1
- data/lib/net/connector.rb +1 -6
- metadata +54 -38
- data/lib/net/connector/device/interface_description.rb +0 -44
- data/lib/net/connector/device/interface_name.rb +0 -56
- data/lib/net/connector/engine/base.rb +0 -4
- data/lib/net/connector/engine/log_messages.rb +0 -47
- data/lib/net/connector/engine/profile.rb +0 -4
- data/lib/net/connector/engine.rb +0 -4
- data/lib/net/connector/operations/local_backup.rb +0 -49
- data/lib/net/connector/operations/parse_output.rb +0 -60
- data/lib/net/connector/operations/private_file.rb +0 -25
- data/lib/net/connector/operations/running_config/cisco.rb +0 -14
- data/lib/net/connector/operations/running_config/hillstone.rb +0 -12
- data/lib/net/connector/operations/running_config/palo_alto.rb +0 -12
- data/lib/net/connector/operations/running_config/strategy.rb +0 -3
- data/lib/net/connector/operations/running_config.rb +0 -16
- data/lib/net/connector/operations/tftp/cisco_ios.rb +0 -11
- data/lib/net/connector/operations/tftp/cisco_nxos.rb +0 -11
- data/lib/net/connector/operations/tftp/file_upload.rb +0 -42
- data/lib/net/connector/operations/tftp/h3c.rb +0 -11
- data/lib/net/connector/operations/tftp/hillstone.rb +0 -11
- data/lib/net/connector/operations/tftp/huawei.rb +0 -11
- data/lib/net/connector/operations/tftp/palo_alto.rb +0 -11
- data/lib/net/connector/operations/tftp/radware.rb +0 -11
- data/lib/net/connector/operations/tftp/strategy.rb +0 -75
- data/lib/net/connector/operations/tftp_backup.rb +0 -120
- data/lib/net/connector/operations/topology/cisco.rb +0 -11
- data/lib/net/connector/operations/topology/h3c.rb +0 -11
- data/lib/net/connector/operations/topology/hillstone.rb +0 -11
- data/lib/net/connector/operations/topology/palo_alto.rb +0 -11
- data/lib/net/connector/operations/topology/radware.rb +0 -11
- data/lib/net/connector/operations/topology/strategy.rb +0 -67
- data/lib/net/connector/operations/topology.rb +0 -193
- data/lib/net/connector/operations.rb +0 -20
|
@@ -12,20 +12,33 @@ module Net
|
|
|
12
12
|
host_overrides: {}, mappings: [])
|
|
13
13
|
@include_hosts = string_list(include_hosts)
|
|
14
14
|
@exclude_hosts = string_list(exclude_hosts)
|
|
15
|
+
@include_vendors = vendor_list(include_vendors)
|
|
16
|
+
@vendor_overrides = vendor_mapping(vendor_overrides)
|
|
17
|
+
@host_overrides = host_mapping(host_overrides)
|
|
18
|
+
@mappings = device_mappings(mappings)
|
|
19
|
+
end
|
|
20
|
+
|
|
21
|
+
# 校验允许使用的厂商列表。
|
|
22
|
+
def vendor_list(include_vendors)
|
|
15
23
|
raise ArgumentError, "include_vendors must be an Array" unless include_vendors.is_a?(Array)
|
|
16
24
|
|
|
17
|
-
|
|
25
|
+
vendors = include_vendors.map do |vendor|
|
|
18
26
|
raise ArgumentError, "include_vendors contains an invalid value" unless vendor.is_a?(String) || vendor.is_a?(Symbol)
|
|
19
27
|
|
|
20
28
|
vendor.to_sym
|
|
21
29
|
end.freeze
|
|
22
|
-
unless
|
|
30
|
+
unless vendors.all? { |vendor| Net::Connector.vendors.include?(vendor) }
|
|
23
31
|
raise ArgumentError, "include_vendors contains an unsupported connector"
|
|
24
32
|
end
|
|
33
|
+
vendors
|
|
34
|
+
end
|
|
35
|
+
|
|
36
|
+
# 将清单厂商标签映射为受支持的连接器。
|
|
37
|
+
def vendor_mapping(vendor_overrides)
|
|
25
38
|
unless vendor_overrides.is_a?(Hash)
|
|
26
39
|
raise ArgumentError, "vendor_overrides must be a Hash"
|
|
27
40
|
end
|
|
28
|
-
|
|
41
|
+
vendor_overrides.to_h do |label, vendor|
|
|
29
42
|
unless label.is_a?(String) && vendor.is_a?(String) && !label.empty?
|
|
30
43
|
raise ArgumentError, "vendor_overrides must map labels to connector names"
|
|
31
44
|
end
|
|
@@ -35,9 +48,13 @@ module Net
|
|
|
35
48
|
|
|
36
49
|
[key, value]
|
|
37
50
|
end.freeze
|
|
51
|
+
end
|
|
52
|
+
|
|
53
|
+
# 按规范化管理地址覆盖厂商,拒绝网段和无效地址。
|
|
54
|
+
def host_mapping(host_overrides)
|
|
38
55
|
raise ArgumentError, "host_overrides must be a Hash" unless host_overrides.is_a?(Hash)
|
|
39
56
|
|
|
40
|
-
|
|
57
|
+
host_overrides.to_h do |host, vendor|
|
|
41
58
|
unless host.is_a?(String) && !host.include?("/") && vendor.is_a?(String)
|
|
42
59
|
raise ArgumentError, "host_overrides must map IP addresses to connector names"
|
|
43
60
|
end
|
|
@@ -51,9 +68,13 @@ module Net
|
|
|
51
68
|
|
|
52
69
|
[address, connector]
|
|
53
70
|
end.freeze
|
|
71
|
+
end
|
|
72
|
+
|
|
73
|
+
# 型号映射仅接受已声明字段,并冻结每条规则。
|
|
74
|
+
def device_mappings(mappings)
|
|
54
75
|
raise ArgumentError, "mappings must be an Array" unless mappings.is_a?(Array)
|
|
55
76
|
|
|
56
|
-
|
|
77
|
+
mappings.map do |rule|
|
|
57
78
|
unless rule.is_a?(Hash) && rule["vendor"].is_a?(String) && !rule["vendor"].empty? &&
|
|
58
79
|
rule["connector"].is_a?(String) &&
|
|
59
80
|
(rule.keys - %w[vendor os model_prefix connector]).empty? &&
|
|
@@ -68,6 +89,8 @@ module Net
|
|
|
68
89
|
end.freeze
|
|
69
90
|
end
|
|
70
91
|
|
|
92
|
+
private :vendor_list, :vendor_mapping, :host_mapping, :device_mappings
|
|
93
|
+
|
|
71
94
|
# 根据地址、显式规则和厂商信息识别连接器。
|
|
72
95
|
def resolve(row)
|
|
73
96
|
begin
|
|
@@ -1,41 +1,95 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
3
|
require "json"
|
|
4
|
-
require "uri"
|
|
5
4
|
|
|
6
5
|
module Net
|
|
7
6
|
module Connector
|
|
8
7
|
module Netdisco
|
|
9
|
-
#
|
|
8
|
+
# 活的凭据来源与不可变的批次策略分开保存;只有明确列出的非敏感字段进入快照。
|
|
10
9
|
class Settings
|
|
11
10
|
DEFAULT_BACKUP_DIRECTORY = "backups"
|
|
12
11
|
DEFAULT_CONCURRENCY = 4
|
|
13
12
|
MAX_CONCURRENCY = 50
|
|
14
|
-
DEFAULT_PAGE_SIZE =
|
|
13
|
+
DEFAULT_PAGE_SIZE = Client::DEFAULTS.fetch(:page_size)
|
|
14
|
+
UNSET = Object.new.freeze
|
|
15
|
+
private_constant :UNSET
|
|
15
16
|
|
|
16
|
-
# 加载 YAML 配置,并使环境变量覆盖文件配置。
|
|
17
17
|
def self.from_file(path, env: ENV)
|
|
18
|
-
new(env: ConfigFile.load(path)
|
|
18
|
+
new(env: env, defaults: ConfigFile.load(path))
|
|
19
19
|
end
|
|
20
20
|
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
@
|
|
21
|
+
def initialize(env: ENV, defaults: {}, overrides: {}, policy: nil, fixed: false)
|
|
22
|
+
@env, @defaults, @overrides = env, defaults, overrides
|
|
23
|
+
@policy, @fixed = policy, fixed
|
|
24
24
|
end
|
|
25
25
|
|
|
26
|
-
#
|
|
27
|
-
def
|
|
28
|
-
|
|
29
|
-
if
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
26
|
+
# 快照自身不保留 ENV、凭据 resolver 或原 Settings 的引用,不能携带秘密进入摘要。
|
|
27
|
+
def snapshot(mode: :backup)
|
|
28
|
+
return @policy.validate!(mode: mode) if @policy
|
|
29
|
+
return validate!(mode: mode) if @fixed
|
|
30
|
+
|
|
31
|
+
keys = mode == :export ? ["NET_CONNECTOR_BACKUP_DIRECTORY"] : non_secret_keys
|
|
32
|
+
values = keys.each_with_object({}) do |key, result|
|
|
33
|
+
item = raw(key, UNSET)
|
|
34
|
+
result[key.freeze] = item.is_a?(String) ? item.dup.freeze : item unless item.equal?(UNSET)
|
|
34
35
|
end
|
|
35
|
-
|
|
36
|
+
self.class.new(env: values.freeze, fixed: true).validate!(mode: mode).freeze
|
|
37
|
+
end
|
|
38
|
+
|
|
39
|
+
# CLI 的一次预览/执行使用同一策略;凭据仍指向原环境映射,允许逐设备轮换。
|
|
40
|
+
def for_run(mode: :backup)
|
|
41
|
+
self.class.new(env: @env, defaults: @defaults, overrides: @overrides, policy: snapshot(mode: mode))
|
|
42
|
+
end
|
|
43
|
+
|
|
44
|
+
def validate!(mode:)
|
|
45
|
+
unless %i[backup tftp inventory show_config export].include?(mode)
|
|
46
|
+
raise ArgumentError, "unknown settings validation mode"
|
|
47
|
+
end
|
|
48
|
+
backup_directory
|
|
49
|
+
return self if mode == :export
|
|
50
|
+
|
|
51
|
+
client_options
|
|
52
|
+
rules
|
|
53
|
+
concurrency
|
|
54
|
+
limit_per_vendor
|
|
55
|
+
log_directory
|
|
56
|
+
connection_options
|
|
57
|
+
Net::Connector.vendors.each { |vendor| connection_options(vendor) }
|
|
58
|
+
tftp_source_files.each_value { |source| TftpTarget.validate_source_file!(source) }
|
|
59
|
+
tftp_vrfs
|
|
60
|
+
TftpTarget.new(host: tftp_server, path: "preflight.cfg") if tftp_server
|
|
61
|
+
self
|
|
62
|
+
end
|
|
63
|
+
|
|
64
|
+
def client(policy: snapshot(mode: :inventory))
|
|
65
|
+
options = policy.client_options
|
|
66
|
+
raise ArgumentError, "NETDISCO_URL is required" unless options.fetch(:url)
|
|
67
|
+
|
|
68
|
+
authentication = if (token = secret_value("NETDISCO_API_KEY"))
|
|
69
|
+
{ api_key: token }
|
|
70
|
+
else
|
|
71
|
+
{ username: required_secret("NETDISCO_USERNAME"), password: required_secret("NETDISCO_PASSWORD") }
|
|
72
|
+
end
|
|
73
|
+
Client.new(**options, **authentication)
|
|
74
|
+
end
|
|
75
|
+
|
|
76
|
+
# 原生客户端和 --show-config 共享同一预算、地址与 HTTP 策略校验。
|
|
77
|
+
def client_options
|
|
78
|
+
options = Client::DEFAULTS.to_h do |name, default|
|
|
79
|
+
key = "NETDISCO_#{name.to_s.upcase}"
|
|
80
|
+
parsed = case name
|
|
81
|
+
when :inventory_timeout then number(key, default)
|
|
82
|
+
when :allow_insecure_http then boolean(key, default)
|
|
83
|
+
else integer(key, default)
|
|
84
|
+
end
|
|
85
|
+
[name, parsed]
|
|
86
|
+
end
|
|
87
|
+
options = Client.options(**options)
|
|
88
|
+
url = value("NETDISCO_URL")
|
|
89
|
+
Client.validate_url!(url, allow_insecure_http: options.fetch(:allow_insecure_http)) if url
|
|
90
|
+
options.merge(url: url).freeze
|
|
36
91
|
end
|
|
37
92
|
|
|
38
|
-
# 读取清单筛选和厂商映射规则。
|
|
39
93
|
def rules
|
|
40
94
|
Rules.new(include_hosts: list("NET_CONNECTOR_INCLUDE_HOSTS"),
|
|
41
95
|
exclude_hosts: list("NET_CONNECTOR_EXCLUDE_HOSTS"),
|
|
@@ -45,15 +99,13 @@ module Net
|
|
|
45
99
|
mappings: json("NET_CONNECTOR_DEVICE_RULES", []))
|
|
46
100
|
end
|
|
47
101
|
|
|
48
|
-
# 读取并展开本地备份目录。
|
|
49
102
|
def backup_directory
|
|
50
|
-
path =
|
|
103
|
+
path = raw("NET_CONNECTOR_BACKUP_DIRECTORY", DEFAULT_BACKUP_DIRECTORY)
|
|
51
104
|
raise ArgumentError, "NET_CONNECTOR_BACKUP_DIRECTORY must be nonempty" unless present?(path)
|
|
52
105
|
|
|
53
106
|
File.expand_path(path)
|
|
54
107
|
end
|
|
55
108
|
|
|
56
|
-
# 读取并限制设备任务并发数。
|
|
57
109
|
def concurrency
|
|
58
110
|
count = integer("NET_CONNECTOR_CONCURRENCY", DEFAULT_CONCURRENCY)
|
|
59
111
|
raise ArgumentError, "NET_CONNECTOR_CONCURRENCY must be at most #{MAX_CONCURRENCY}" if count > MAX_CONCURRENCY
|
|
@@ -61,121 +113,172 @@ module Net
|
|
|
61
113
|
count
|
|
62
114
|
end
|
|
63
115
|
|
|
64
|
-
# 确定厂商采样数量:本地默认全量,TFTP 默认每厂商五台。
|
|
65
116
|
def limit_per_vendor(tftp: false)
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
117
|
+
count = if configured?("NET_CONNECTOR_SAMPLE_PER_VENDOR")
|
|
118
|
+
integer("NET_CONNECTOR_SAMPLE_PER_VENDOR", 5) unless raw("NET_CONNECTOR_SAMPLE_PER_VENDOR").nil?
|
|
119
|
+
elsif tftp
|
|
120
|
+
5
|
|
121
|
+
end
|
|
122
|
+
Planner.validate_limit!(count)
|
|
123
|
+
count
|
|
69
124
|
end
|
|
70
125
|
|
|
71
|
-
# 读取厂商的 TFTP 源文件配置,并提供华为启动配置默认路径。
|
|
72
126
|
def tftp_source_files
|
|
73
|
-
files = { huawei:
|
|
127
|
+
files = { huawei: raw("NET_CONNECTOR_HUAWEI_TFTP_SOURCE_FILE", "flash:/startup.cfg") }
|
|
74
128
|
%i[h3c h3c_wireless].each do |vendor|
|
|
75
129
|
key = "NET_CONNECTOR_#{vendor.to_s.upcase}_TFTP_SOURCE_FILE"
|
|
76
|
-
files[vendor] =
|
|
130
|
+
files[vendor] = raw(key) if configured?(key)
|
|
77
131
|
end
|
|
78
132
|
files
|
|
79
133
|
end
|
|
80
134
|
|
|
81
|
-
# 读取各厂商 TFTP 出口 VRF 的名称映射。
|
|
82
135
|
def tftp_vrfs
|
|
83
|
-
|
|
84
|
-
raise ArgumentError, "NET_CONNECTOR_TFTP_VRFS must be a JSON object" unless
|
|
136
|
+
mapping = json("NET_CONNECTOR_TFTP_VRFS", {})
|
|
137
|
+
raise ArgumentError, "NET_CONNECTOR_TFTP_VRFS must be a JSON object" unless mapping.is_a?(Hash)
|
|
85
138
|
|
|
86
|
-
|
|
139
|
+
self.class.validate_vrfs!(mapping.transform_keys(&:to_sym))
|
|
140
|
+
end
|
|
141
|
+
|
|
142
|
+
def self.validate_vrfs!(vrfs)
|
|
143
|
+
unless vrfs.is_a?(Hash) && vrfs.all? { |vendor, name|
|
|
144
|
+
%i[cisco_nxos hillstone].include?(vendor) && name.is_a?(String) &&
|
|
145
|
+
name.match?(/\A[A-Za-z0-9_][A-Za-z0-9_.-]*\z/)
|
|
146
|
+
}
|
|
147
|
+
raise ArgumentError, "vrfs must map supported vendor names to safe VRF names"
|
|
148
|
+
end
|
|
149
|
+
vrfs
|
|
87
150
|
end
|
|
88
151
|
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
def
|
|
152
|
+
def tftp_server = value("TFTP_HOST")
|
|
153
|
+
|
|
154
|
+
def connection_options(vendor = nil)
|
|
155
|
+
prefix = "NET_CONNECTOR_#{vendor.to_s.upcase}_"
|
|
156
|
+
configuration = Configuration.new(
|
|
157
|
+
protocol: (vendor && value(prefix + "PROTOCOL")) || value("NET_CONNECTOR_PROTOCOL") || "ssh",
|
|
158
|
+
log_level: value("NET_CONNECTOR_LOG_LEVEL") || "info", known_hosts: value("NET_CONNECTOR_KNOWN_HOSTS"),
|
|
159
|
+
host_key_policy: value("NET_CONNECTOR_HOST_KEY_POLICY") || "strict",
|
|
160
|
+
max_script_output_bytes: optional_integer("NET_CONNECTOR_MAX_SCRIPT_OUTPUT_BYTES")
|
|
161
|
+
)
|
|
162
|
+
options = %i[protocol log_level known_hosts host_key_policy].to_h { |key| [key, configuration.public_send(key)] }
|
|
163
|
+
options[:max_script_output_bytes] = configuration.max_script_output_bytes if configuration.max_script_output_bytes
|
|
164
|
+
options.freeze
|
|
165
|
+
end
|
|
166
|
+
|
|
167
|
+
def device_credentials_for(device)
|
|
92
168
|
prefix = "NET_CONNECTOR_#{device.vendor.to_s.upcase}_"
|
|
93
|
-
username =
|
|
169
|
+
username = secret_value(prefix + "USERNAME") || secret_value("NET_CONNECTOR_DEVICE_USERNAME")
|
|
94
170
|
return nil unless username
|
|
95
171
|
|
|
96
|
-
{
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
enable_password: value(prefix + "ENABLE_PASSWORD") || value("NET_CONNECTOR_ENABLE_PASSWORD"),
|
|
100
|
-
protocol: (value(prefix + "PROTOCOL") || value("NET_CONNECTOR_PROTOCOL") || "ssh").to_sym,
|
|
101
|
-
log_level: (value("NET_CONNECTOR_LOG_LEVEL") || "info").to_sym,
|
|
102
|
-
known_hosts: value("NET_CONNECTOR_KNOWN_HOSTS"),
|
|
103
|
-
host_key_policy: (value("NET_CONNECTOR_HOST_KEY_POLICY") || "strict").to_sym
|
|
104
|
-
}
|
|
172
|
+
{ username: username,
|
|
173
|
+
password: secret_value(prefix + "PASSWORD") || secret_value("NET_CONNECTOR_DEVICE_PASSWORD"),
|
|
174
|
+
enable_password: secret_value(prefix + "ENABLE_PASSWORD") || secret_value("NET_CONNECTOR_ENABLE_PASSWORD") }
|
|
105
175
|
end
|
|
106
176
|
|
|
107
|
-
# 读取并展开设备日志目录。
|
|
108
177
|
def log_directory
|
|
109
178
|
path = value("NET_CONNECTOR_LOG_DIRECTORY")
|
|
110
179
|
File.expand_path(path) if path
|
|
111
180
|
end
|
|
112
181
|
|
|
113
|
-
# 仅展示非敏感配置,不输出设备凭据或 API 密钥。
|
|
114
|
-
# 生成不包含凭据的可展示配置。
|
|
115
182
|
def public_config
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
183
|
+
snapshot(mode: :show_config).send(:config_hash)
|
|
184
|
+
end
|
|
185
|
+
|
|
186
|
+
def inspect = "#<#{self.class}>"
|
|
187
|
+
|
|
188
|
+
private
|
|
189
|
+
|
|
190
|
+
def config_hash
|
|
124
191
|
{
|
|
125
|
-
netdisco:
|
|
126
|
-
backup: { directory: backup_directory, concurrency: concurrency,
|
|
127
|
-
limit_per_vendor: limit_per_vendor },
|
|
192
|
+
netdisco: client_options,
|
|
193
|
+
backup: { directory: backup_directory, concurrency: concurrency, limit_per_vendor: limit_per_vendor },
|
|
128
194
|
inventory: { include_hosts: list("NET_CONNECTOR_INCLUDE_HOSTS"),
|
|
129
195
|
exclude_hosts: list("NET_CONNECTOR_EXCLUDE_HOSTS"),
|
|
130
196
|
include_vendors: list("NET_CONNECTOR_INCLUDE_VENDORS"),
|
|
131
197
|
vendor_overrides: json("NET_CONNECTOR_VENDOR_OVERRIDES", {}),
|
|
132
198
|
host_overrides: json("NET_CONNECTOR_HOST_OVERRIDES", {}),
|
|
133
199
|
device_rules: json("NET_CONNECTOR_DEVICE_RULES", []) },
|
|
134
|
-
ssh: {
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
200
|
+
ssh: connection_options.transform_values { |item| item.is_a?(Symbol) ? item.to_s : item }.merge(
|
|
201
|
+
log_directory: log_directory,
|
|
202
|
+
vendor_protocols: Net::Connector.vendors.filter_map do |vendor|
|
|
203
|
+
protocol = value("NET_CONNECTOR_#{vendor.to_s.upcase}_PROTOCOL")
|
|
204
|
+
[vendor, protocol] if protocol
|
|
205
|
+
end.to_h
|
|
206
|
+
),
|
|
207
|
+
tftp: { server: tftp_server, vrfs: tftp_vrfs.transform_keys(&:to_s), source_files: tftp_source_files }
|
|
140
208
|
}
|
|
141
|
-
rescue URI::InvalidURIError
|
|
142
|
-
raise ArgumentError, "NETDISCO_URL is invalid"
|
|
143
209
|
end
|
|
144
210
|
|
|
145
|
-
|
|
211
|
+
def non_secret_keys
|
|
212
|
+
keys = ConfigFile::FIELDS.values.flat_map { |section| section.values.map(&:first) }
|
|
213
|
+
keys + Net::Connector.vendors.map { |vendor| "NET_CONNECTOR_#{vendor.to_s.upcase}_PROTOCOL" }
|
|
214
|
+
end
|
|
215
|
+
|
|
216
|
+
def raw(key, default = nil)
|
|
217
|
+
return @policy.send(:raw, key, default) if @policy
|
|
218
|
+
return @overrides.fetch(key) if @overrides.key?(key)
|
|
219
|
+
return @env.fetch(key) if @env.key?(key)
|
|
220
|
+
|
|
221
|
+
@defaults.fetch(key, default)
|
|
222
|
+
end
|
|
223
|
+
|
|
224
|
+
def configured?(key) = !raw(key, UNSET).equal?(UNSET)
|
|
146
225
|
|
|
147
|
-
# 解析指定环境变量中的 JSON 配置。
|
|
148
226
|
def json(key, default)
|
|
149
|
-
|
|
150
|
-
|
|
227
|
+
text = value(key)
|
|
228
|
+
text ? JSON.parse(text) : default
|
|
151
229
|
rescue JSON::ParserError
|
|
152
|
-
raise ArgumentError, "#{key} must be valid JSON"
|
|
230
|
+
raise ArgumentError, "#{key} must be valid JSON", cause: nil
|
|
153
231
|
end
|
|
154
232
|
|
|
155
|
-
|
|
156
|
-
def list(key) = @env.fetch(key, "").split(",").map(&:strip).reject(&:empty?)
|
|
233
|
+
def list(key) = (value(key) || "").split(",").map(&:strip).reject(&:empty?)
|
|
157
234
|
|
|
158
|
-
# 读取正整数配置并统一错误信息。
|
|
159
235
|
def integer(key, default)
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
raise ArgumentError, "#{key} must be a positive integer" unless value.positive?
|
|
236
|
+
result = Integer(raw(key, default).to_s, 10)
|
|
237
|
+
raise ArgumentError unless result.positive?
|
|
163
238
|
|
|
164
|
-
|
|
239
|
+
result
|
|
165
240
|
rescue ArgumentError, TypeError
|
|
166
|
-
raise ArgumentError, "#{key} must be a positive integer"
|
|
241
|
+
raise ArgumentError, "#{key} must be a positive integer", cause: nil
|
|
242
|
+
end
|
|
243
|
+
|
|
244
|
+
def optional_integer(key)
|
|
245
|
+
integer(key, nil) unless raw(key).nil?
|
|
246
|
+
end
|
|
247
|
+
|
|
248
|
+
def number(key, default)
|
|
249
|
+
input = raw(key, default)
|
|
250
|
+
result = input.is_a?(String) ? Float(input) : input
|
|
251
|
+
unless result.is_a?(Numeric) && result.real? && result.finite? && result.positive? && result.to_f.finite?
|
|
252
|
+
raise ArgumentError
|
|
253
|
+
end
|
|
254
|
+
result
|
|
255
|
+
rescue ArgumentError, TypeError
|
|
256
|
+
raise ArgumentError, "#{key} must be a positive finite number", cause: nil
|
|
257
|
+
end
|
|
258
|
+
|
|
259
|
+
def boolean(key, default)
|
|
260
|
+
result = raw(key, default)
|
|
261
|
+
return true if result == true || result == "true"
|
|
262
|
+
return false if result == false || result == "false"
|
|
263
|
+
|
|
264
|
+
raise ArgumentError, "#{key} must be true or false"
|
|
167
265
|
end
|
|
168
266
|
|
|
169
|
-
# 读取非空字符串配置。
|
|
170
267
|
def value(key)
|
|
171
|
-
result =
|
|
268
|
+
result = raw(key)
|
|
269
|
+
raise ArgumentError, "#{key} must be a String" unless result.nil? || result.is_a?(String)
|
|
270
|
+
|
|
172
271
|
result if present?(result)
|
|
173
272
|
end
|
|
174
273
|
|
|
175
|
-
|
|
176
|
-
|
|
274
|
+
def secret_value(key)
|
|
275
|
+
result = @env[key]
|
|
276
|
+
raise ArgumentError, "#{key} must be a String" unless result.nil? || result.is_a?(String)
|
|
277
|
+
|
|
278
|
+
result.dup.freeze if present?(result)
|
|
279
|
+
end
|
|
177
280
|
|
|
178
|
-
#
|
|
281
|
+
def required_secret(key) = secret_value(key) || (raise ArgumentError, "#{key} is required")
|
|
179
282
|
def present?(value) = value.is_a?(String) && !value.strip.empty?
|
|
180
283
|
end
|
|
181
284
|
end
|
|
@@ -31,25 +31,10 @@ module Net
|
|
|
31
31
|
Thread.current.report_on_exception = false
|
|
32
32
|
while (task = queue.pop)
|
|
33
33
|
index, device = task
|
|
34
|
-
|
|
35
|
-
begin
|
|
36
|
-
on_start&.call(device)
|
|
37
|
-
rescue StandardError => error
|
|
38
|
-
callback_errors << { host: device.host, error_type: error.class.name }.freeze
|
|
39
|
-
end
|
|
40
|
-
result = begin
|
|
41
|
-
yield device
|
|
42
|
-
rescue StandardError => error
|
|
43
|
-
on_error.call(device, error)
|
|
44
|
-
end
|
|
45
|
-
result = result.with(started_at: started_at, finished_at: Time.now.utc)
|
|
34
|
+
result = run_task(device, on_error, on_start, callback_errors) { yield device }
|
|
46
35
|
outcomes[index] = result
|
|
47
36
|
Array(on_result).each do |callback|
|
|
48
|
-
|
|
49
|
-
callback.call(result)
|
|
50
|
-
rescue StandardError => error
|
|
51
|
-
callback_errors << { host: device.host, error_type: error.class.name }.freeze
|
|
52
|
-
end
|
|
37
|
+
notify(callback, result, host: device.host, errors: callback_errors)
|
|
53
38
|
end
|
|
54
39
|
end
|
|
55
40
|
ensure
|
|
@@ -76,6 +61,30 @@ module Net
|
|
|
76
61
|
end
|
|
77
62
|
Array.new(callback_errors.size) { callback_errors.pop }.freeze
|
|
78
63
|
end
|
|
64
|
+
|
|
65
|
+
private
|
|
66
|
+
|
|
67
|
+
# 普通设备故障转换为结果,中断仍交给批次统一停止其余线程。
|
|
68
|
+
def run_task(device, on_error, on_start, callback_errors)
|
|
69
|
+
started_at = Time.now.utc
|
|
70
|
+
started = monotonic
|
|
71
|
+
notify(on_start, device, host: device.host, errors: callback_errors) unless on_start.nil?
|
|
72
|
+
result = begin
|
|
73
|
+
yield
|
|
74
|
+
rescue StandardError => error
|
|
75
|
+
on_error.call(device, error)
|
|
76
|
+
end
|
|
77
|
+
result.with(started_at: started_at, finished_at: Time.now.utc, duration_ms: ((monotonic - started) * 1000).round)
|
|
78
|
+
end
|
|
79
|
+
|
|
80
|
+
def monotonic = Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
|
81
|
+
|
|
82
|
+
# 回调故障不覆盖设备结果,也不把异常消息中的凭据写入报告。
|
|
83
|
+
def notify(callback, value, host:, errors:)
|
|
84
|
+
callback.call(value)
|
|
85
|
+
rescue StandardError => error
|
|
86
|
+
errors << { host: host, error_type: error.class.name }.freeze
|
|
87
|
+
end
|
|
79
88
|
end
|
|
80
89
|
end
|
|
81
90
|
end
|
|
@@ -0,0 +1,116 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "digest"
|
|
4
|
+
require_relative "../engine/errors"
|
|
5
|
+
|
|
6
|
+
module Net
|
|
7
|
+
module Connector
|
|
8
|
+
class BackupBusy < Error; end
|
|
9
|
+
|
|
10
|
+
module Storage
|
|
11
|
+
# 路径锁在设备采集之前取得;锁文件长期保留,避免等待者落在已经删除的 inode 上。
|
|
12
|
+
class BackupLock
|
|
13
|
+
STORAGE_KEY = :net_connector_backup_delegation
|
|
14
|
+
private_constant :STORAGE_KEY
|
|
15
|
+
|
|
16
|
+
attr_reader :path
|
|
17
|
+
|
|
18
|
+
def self.lock_path(path)
|
|
19
|
+
destination = File.expand_path(path)
|
|
20
|
+
directory = File.realpath(File.dirname(destination))
|
|
21
|
+
# 锁名先折叠 Unicode 与大小写,避免大小写不敏感文件系统上的别名绕过互斥。
|
|
22
|
+
# 在区分大小写的文件系统上可能保守地合并两个锁,但不会改写目标文件名。
|
|
23
|
+
filename = File.basename(destination).b.force_encoding(Encoding::UTF_8)
|
|
24
|
+
filename = filename.valid_encoding? ? filename.unicode_normalize(:nfc).downcase(:fold).unicode_normalize(:nfc) : filename.b.downcase
|
|
25
|
+
File.join(directory, ".net-connector-#{Digest::SHA256.hexdigest(filename)}.lock")
|
|
26
|
+
end
|
|
27
|
+
|
|
28
|
+
def self.synchronize(path, **options, &block) = new(path, **options).synchronize(&block)
|
|
29
|
+
|
|
30
|
+
def initialize(path, timeout: 0, host: nil)
|
|
31
|
+
raise ArgumentError, "path must be a nonempty String" unless path.is_a?(String) && !path.empty?
|
|
32
|
+
unless timeout.is_a?(Numeric) && timeout.real? && timeout.finite? && timeout >= 0 && timeout.to_f.finite?
|
|
33
|
+
raise ArgumentError, "lock_timeout must be a nonnegative finite number"
|
|
34
|
+
end
|
|
35
|
+
|
|
36
|
+
@path = File.expand_path(path).freeze
|
|
37
|
+
@lock_path = self.class.lock_path(@path).freeze
|
|
38
|
+
@timeout, @host = timeout.to_f, host
|
|
39
|
+
end
|
|
40
|
+
|
|
41
|
+
def synchronize
|
|
42
|
+
# Fleet 只向下层备份借用一次;消费后清除,采集回调中的递归备份仍须竞争锁。
|
|
43
|
+
# Thread#[] 是 Fiber 局部存储;fork 后不能消费父进程的借用记录。
|
|
44
|
+
if Thread.current[STORAGE_KEY] == [@lock_path, Process.pid]
|
|
45
|
+
Thread.current[STORAGE_KEY] = nil
|
|
46
|
+
return yield self
|
|
47
|
+
end
|
|
48
|
+
|
|
49
|
+
file = nil
|
|
50
|
+
begin
|
|
51
|
+
file = open_lock
|
|
52
|
+
verify_lock!(file)
|
|
53
|
+
acquire(file)
|
|
54
|
+
verify_lock!(file)
|
|
55
|
+
@owner = [Process.pid, Fiber.current]
|
|
56
|
+
yield self
|
|
57
|
+
ensure
|
|
58
|
+
@owner = nil
|
|
59
|
+
file&.close
|
|
60
|
+
end
|
|
61
|
+
end
|
|
62
|
+
|
|
63
|
+
# 仅持有实际 flock 的同一 Fiber 可授权一次内层采集,避免把路径锁变成任意可重入锁。
|
|
64
|
+
def with_delegated_lock
|
|
65
|
+
raise ThreadError, "backup path delegation requires lock ownership" unless @owner == [Process.pid, Fiber.current]
|
|
66
|
+
|
|
67
|
+
previous = Thread.current[STORAGE_KEY]
|
|
68
|
+
begin
|
|
69
|
+
Thread.current[STORAGE_KEY] = [@lock_path, Process.pid]
|
|
70
|
+
yield
|
|
71
|
+
ensure
|
|
72
|
+
Thread.current[STORAGE_KEY] = previous
|
|
73
|
+
end
|
|
74
|
+
end
|
|
75
|
+
|
|
76
|
+
private
|
|
77
|
+
|
|
78
|
+
def open_lock
|
|
79
|
+
File.open(@lock_path, File::RDWR | File::CREAT | File::NOFOLLOW | File::NONBLOCK, 0o600)
|
|
80
|
+
rescue Errno::ELOOP, Errno::EISDIR, Errno::ENXIO
|
|
81
|
+
raise ArgumentError, "backup lock must be a private regular file", cause: nil
|
|
82
|
+
end
|
|
83
|
+
|
|
84
|
+
def verify_lock!(file)
|
|
85
|
+
stat = file.stat
|
|
86
|
+
unless stat.file? && stat.uid == Process.euid && stat.nlink == 1 && (stat.mode & 0o7777) == 0o600
|
|
87
|
+
raise ArgumentError, "backup lock must be an owned regular file with mode 0600 and one link"
|
|
88
|
+
end
|
|
89
|
+
current = File.lstat(@lock_path)
|
|
90
|
+
unless current.file? && current.dev == stat.dev && current.ino == stat.ino
|
|
91
|
+
raise ArgumentError, "backup lock path changed while acquiring ownership"
|
|
92
|
+
end
|
|
93
|
+
rescue Errno::ENOENT
|
|
94
|
+
raise ArgumentError, "backup lock path changed while acquiring ownership", cause: nil
|
|
95
|
+
end
|
|
96
|
+
|
|
97
|
+
def acquire(file)
|
|
98
|
+
deadline = monotonic + @timeout
|
|
99
|
+
loop do
|
|
100
|
+
acquired = file.flock(File::LOCK_EX | File::LOCK_NB)
|
|
101
|
+
return if acquired && (@timeout.zero? || monotonic < deadline)
|
|
102
|
+
|
|
103
|
+
remaining = deadline - monotonic
|
|
104
|
+
if remaining <= 0
|
|
105
|
+
raise BackupBusy.new("backup path already belongs to another operation", host: @host, phase: :backup), cause: nil
|
|
106
|
+
end
|
|
107
|
+
wait([remaining, 0.05].min)
|
|
108
|
+
end
|
|
109
|
+
end
|
|
110
|
+
|
|
111
|
+
def monotonic = Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
|
112
|
+
def wait(seconds) = sleep(seconds)
|
|
113
|
+
end
|
|
114
|
+
end
|
|
115
|
+
end
|
|
116
|
+
end
|