net-connector 0.4.1 → 0.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (107) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +50 -0
  3. data/CONTRIBUTING.md +40 -0
  4. data/README.md +93 -18
  5. data/SECURITY.md +11 -0
  6. data/docs/VERIFICATION.md +139 -6
  7. data/docs/architecture.md +248 -39
  8. data/lib/net/connector/device/base.rb +28 -101
  9. data/lib/net/connector/device/local_backup.rb +96 -0
  10. data/lib/net/connector/device/profile.rb +8 -3
  11. data/lib/net/connector/device/running_config/strategy.rb +1 -1
  12. data/lib/net/connector/device/running_config.rb +46 -5
  13. data/lib/net/connector/device/save_config.rb +21 -0
  14. data/lib/net/connector/device/tftp/file_upload.rb +53 -0
  15. data/lib/net/connector/device/tftp/receipt.rb +71 -0
  16. data/lib/net/connector/device/tftp/strategy.rb +67 -0
  17. data/lib/net/connector/device/tftp/target.rb +63 -0
  18. data/lib/net/connector/device/tftp.rb +117 -0
  19. data/lib/net/connector/device/topology/immediate_strategy.rb +40 -0
  20. data/lib/net/connector/device/topology/interface_description.rb +46 -0
  21. data/lib/net/connector/device/topology/interface_name.rb +58 -0
  22. data/lib/net/connector/device/topology/strategy.rb +75 -0
  23. data/lib/net/connector/device/topology.rb +277 -0
  24. data/lib/net/connector/engine/authentication.rb +1 -1
  25. data/lib/net/connector/engine/command.rb +14 -2
  26. data/lib/net/connector/engine/configuration.rb +11 -8
  27. data/lib/net/connector/engine/dialogue.rb +59 -35
  28. data/lib/net/connector/engine/error_metadata.rb +50 -0
  29. data/lib/net/connector/engine/errors.rb +55 -56
  30. data/lib/net/connector/engine/execution.rb +44 -10
  31. data/lib/net/connector/engine/log/event.rb +66 -0
  32. data/lib/net/connector/engine/log/formatter.rb +16 -0
  33. data/lib/net/connector/engine/log/messages.rb +52 -0
  34. data/lib/net/connector/engine/log/stream.rb +56 -0
  35. data/lib/net/connector/engine/log.rb +107 -84
  36. data/lib/net/connector/engine/session.rb +115 -43
  37. data/lib/net/connector/engine/terminal_renderer.rb +7 -4
  38. data/lib/net/connector/engine/terminal_text.rb +33 -0
  39. data/lib/net/connector/engine/transport.rb +2 -2
  40. data/lib/net/connector/netdisco/batch.rb +25 -33
  41. data/lib/net/connector/netdisco/cli.rb +71 -31
  42. data/lib/net/connector/netdisco/client.rb +206 -73
  43. data/lib/net/connector/netdisco/config_file.rb +26 -4
  44. data/lib/net/connector/netdisco/device.rb +4 -5
  45. data/lib/net/connector/netdisco/diagnostic.rb +61 -0
  46. data/lib/net/connector/netdisco/fleet.rb +96 -66
  47. data/lib/net/connector/netdisco/inventory_budget.rb +49 -0
  48. data/lib/net/connector/netdisco/report.rb +115 -0
  49. data/lib/net/connector/netdisco/result_store.rb +2 -2
  50. data/lib/net/connector/netdisco/rules.rb +28 -5
  51. data/lib/net/connector/netdisco/settings.rb +188 -85
  52. data/lib/net/connector/netdisco/worker.rb +26 -17
  53. data/lib/net/connector/storage/backup_lock.rb +116 -0
  54. data/lib/net/connector/storage/private_file.rb +109 -0
  55. data/lib/net/connector/storage/safe_file.rb +62 -0
  56. data/lib/net/connector/{operations → storage}/saved_config.rb +25 -20
  57. data/lib/net/connector/storage.rb +15 -0
  58. data/lib/net/connector/textfsm.rb +72 -0
  59. data/lib/net/connector/vendor/cisco_ios/tftp_backup.rb +12 -5
  60. data/lib/net/connector/vendor/cisco_ios/topology.rb +12 -6
  61. data/lib/net/connector/vendor/cisco_nxos/tftp_backup.rb +11 -4
  62. data/lib/net/connector/vendor/h3c/tftp_backup.rb +9 -4
  63. data/lib/net/connector/vendor/h3c/topology.rb +11 -6
  64. data/lib/net/connector/vendor/h3c.rb +3 -3
  65. data/lib/net/connector/vendor/hillstone/tftp_backup.rb +15 -6
  66. data/lib/net/connector/vendor/hillstone/topology.rb +11 -6
  67. data/lib/net/connector/vendor/huawei/tftp_backup.rb +9 -3
  68. data/lib/net/connector/vendor/palo_alto/running_config.rb +1 -1
  69. data/lib/net/connector/vendor/palo_alto/tftp_backup.rb +17 -8
  70. data/lib/net/connector/vendor/palo_alto/topology.rb +9 -7
  71. data/lib/net/connector/vendor/radware/tftp_backup.rb +12 -5
  72. data/lib/net/connector/vendor/radware/topology.rb +3 -2
  73. data/lib/net/connector/version.rb +1 -1
  74. data/lib/net/connector.rb +1 -6
  75. metadata +54 -38
  76. data/lib/net/connector/device/interface_description.rb +0 -44
  77. data/lib/net/connector/device/interface_name.rb +0 -56
  78. data/lib/net/connector/engine/base.rb +0 -4
  79. data/lib/net/connector/engine/log_messages.rb +0 -47
  80. data/lib/net/connector/engine/profile.rb +0 -4
  81. data/lib/net/connector/engine.rb +0 -4
  82. data/lib/net/connector/operations/local_backup.rb +0 -49
  83. data/lib/net/connector/operations/parse_output.rb +0 -60
  84. data/lib/net/connector/operations/private_file.rb +0 -25
  85. data/lib/net/connector/operations/running_config/cisco.rb +0 -14
  86. data/lib/net/connector/operations/running_config/hillstone.rb +0 -12
  87. data/lib/net/connector/operations/running_config/palo_alto.rb +0 -12
  88. data/lib/net/connector/operations/running_config/strategy.rb +0 -3
  89. data/lib/net/connector/operations/running_config.rb +0 -16
  90. data/lib/net/connector/operations/tftp/cisco_ios.rb +0 -11
  91. data/lib/net/connector/operations/tftp/cisco_nxos.rb +0 -11
  92. data/lib/net/connector/operations/tftp/file_upload.rb +0 -42
  93. data/lib/net/connector/operations/tftp/h3c.rb +0 -11
  94. data/lib/net/connector/operations/tftp/hillstone.rb +0 -11
  95. data/lib/net/connector/operations/tftp/huawei.rb +0 -11
  96. data/lib/net/connector/operations/tftp/palo_alto.rb +0 -11
  97. data/lib/net/connector/operations/tftp/radware.rb +0 -11
  98. data/lib/net/connector/operations/tftp/strategy.rb +0 -75
  99. data/lib/net/connector/operations/tftp_backup.rb +0 -120
  100. data/lib/net/connector/operations/topology/cisco.rb +0 -11
  101. data/lib/net/connector/operations/topology/h3c.rb +0 -11
  102. data/lib/net/connector/operations/topology/hillstone.rb +0 -11
  103. data/lib/net/connector/operations/topology/palo_alto.rb +0 -11
  104. data/lib/net/connector/operations/topology/radware.rb +0 -11
  105. data/lib/net/connector/operations/topology/strategy.rb +0 -67
  106. data/lib/net/connector/operations/topology.rb +0 -193
  107. data/lib/net/connector/operations.rb +0 -20
@@ -12,20 +12,33 @@ module Net
12
12
  host_overrides: {}, mappings: [])
13
13
  @include_hosts = string_list(include_hosts)
14
14
  @exclude_hosts = string_list(exclude_hosts)
15
+ @include_vendors = vendor_list(include_vendors)
16
+ @vendor_overrides = vendor_mapping(vendor_overrides)
17
+ @host_overrides = host_mapping(host_overrides)
18
+ @mappings = device_mappings(mappings)
19
+ end
20
+
21
+ # 校验允许使用的厂商列表。
22
+ def vendor_list(include_vendors)
15
23
  raise ArgumentError, "include_vendors must be an Array" unless include_vendors.is_a?(Array)
16
24
 
17
- @include_vendors = include_vendors.map do |vendor|
25
+ vendors = include_vendors.map do |vendor|
18
26
  raise ArgumentError, "include_vendors contains an invalid value" unless vendor.is_a?(String) || vendor.is_a?(Symbol)
19
27
 
20
28
  vendor.to_sym
21
29
  end.freeze
22
- unless @include_vendors.all? { |vendor| Net::Connector.vendors.include?(vendor) }
30
+ unless vendors.all? { |vendor| Net::Connector.vendors.include?(vendor) }
23
31
  raise ArgumentError, "include_vendors contains an unsupported connector"
24
32
  end
33
+ vendors
34
+ end
35
+
36
+ # 将清单厂商标签映射为受支持的连接器。
37
+ def vendor_mapping(vendor_overrides)
25
38
  unless vendor_overrides.is_a?(Hash)
26
39
  raise ArgumentError, "vendor_overrides must be a Hash"
27
40
  end
28
- @vendor_overrides = vendor_overrides.to_h do |label, vendor|
41
+ vendor_overrides.to_h do |label, vendor|
29
42
  unless label.is_a?(String) && vendor.is_a?(String) && !label.empty?
30
43
  raise ArgumentError, "vendor_overrides must map labels to connector names"
31
44
  end
@@ -35,9 +48,13 @@ module Net
35
48
 
36
49
  [key, value]
37
50
  end.freeze
51
+ end
52
+
53
+ # 按规范化管理地址覆盖厂商,拒绝网段和无效地址。
54
+ def host_mapping(host_overrides)
38
55
  raise ArgumentError, "host_overrides must be a Hash" unless host_overrides.is_a?(Hash)
39
56
 
40
- @host_overrides = host_overrides.to_h do |host, vendor|
57
+ host_overrides.to_h do |host, vendor|
41
58
  unless host.is_a?(String) && !host.include?("/") && vendor.is_a?(String)
42
59
  raise ArgumentError, "host_overrides must map IP addresses to connector names"
43
60
  end
@@ -51,9 +68,13 @@ module Net
51
68
 
52
69
  [address, connector]
53
70
  end.freeze
71
+ end
72
+
73
+ # 型号映射仅接受已声明字段,并冻结每条规则。
74
+ def device_mappings(mappings)
54
75
  raise ArgumentError, "mappings must be an Array" unless mappings.is_a?(Array)
55
76
 
56
- @mappings = mappings.map do |rule|
77
+ mappings.map do |rule|
57
78
  unless rule.is_a?(Hash) && rule["vendor"].is_a?(String) && !rule["vendor"].empty? &&
58
79
  rule["connector"].is_a?(String) &&
59
80
  (rule.keys - %w[vendor os model_prefix connector]).empty? &&
@@ -68,6 +89,8 @@ module Net
68
89
  end.freeze
69
90
  end
70
91
 
92
+ private :vendor_list, :vendor_mapping, :host_mapping, :device_mappings
93
+
71
94
  # 根据地址、显式规则和厂商信息识别连接器。
72
95
  def resolve(row)
73
96
  begin
@@ -1,41 +1,95 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  require "json"
4
- require "uri"
5
4
 
6
5
  module Net
7
6
  module Connector
8
7
  module Netdisco
9
- # 使用时从环境中读取配置,并提供明确的默认值。
8
+ # 活的凭据来源与不可变的批次策略分开保存;只有明确列出的非敏感字段进入快照。
10
9
  class Settings
11
10
  DEFAULT_BACKUP_DIRECTORY = "backups"
12
11
  DEFAULT_CONCURRENCY = 4
13
12
  MAX_CONCURRENCY = 50
14
- DEFAULT_PAGE_SIZE = 500
13
+ DEFAULT_PAGE_SIZE = Client::DEFAULTS.fetch(:page_size)
14
+ UNSET = Object.new.freeze
15
+ private_constant :UNSET
15
16
 
16
- # 加载 YAML 配置,并使环境变量覆盖文件配置。
17
17
  def self.from_file(path, env: ENV)
18
- new(env: ConfigFile.load(path).merge(env.to_h))
18
+ new(env: env, defaults: ConfigFile.load(path))
19
19
  end
20
20
 
21
- # 保存当前批次读取配置所用的环境映射。
22
- def initialize(env: ENV)
23
- @env = env
21
+ def initialize(env: ENV, defaults: {}, overrides: {}, policy: nil, fixed: false)
22
+ @env, @defaults, @overrides = env, defaults, overrides
23
+ @policy, @fixed = policy, fixed
24
24
  end
25
25
 
26
- # 使用 Netdisco 地址和认证信息创建客户端。
27
- def client
28
- options = { url: required("NETDISCO_URL"), page_size: integer("NETDISCO_PAGE_SIZE", DEFAULT_PAGE_SIZE) }
29
- if present?(@env["NETDISCO_API_KEY"])
30
- options[:api_key] = @env["NETDISCO_API_KEY"]
31
- else
32
- options[:username] = required("NETDISCO_USERNAME")
33
- options[:password] = required("NETDISCO_PASSWORD")
26
+ # 快照自身不保留 ENV、凭据 resolver 或原 Settings 的引用,不能携带秘密进入摘要。
27
+ def snapshot(mode: :backup)
28
+ return @policy.validate!(mode: mode) if @policy
29
+ return validate!(mode: mode) if @fixed
30
+
31
+ keys = mode == :export ? ["NET_CONNECTOR_BACKUP_DIRECTORY"] : non_secret_keys
32
+ values = keys.each_with_object({}) do |key, result|
33
+ item = raw(key, UNSET)
34
+ result[key.freeze] = item.is_a?(String) ? item.dup.freeze : item unless item.equal?(UNSET)
34
35
  end
35
- Client.new(**options)
36
+ self.class.new(env: values.freeze, fixed: true).validate!(mode: mode).freeze
37
+ end
38
+
39
+ # CLI 的一次预览/执行使用同一策略;凭据仍指向原环境映射,允许逐设备轮换。
40
+ def for_run(mode: :backup)
41
+ self.class.new(env: @env, defaults: @defaults, overrides: @overrides, policy: snapshot(mode: mode))
42
+ end
43
+
44
+ def validate!(mode:)
45
+ unless %i[backup tftp inventory show_config export].include?(mode)
46
+ raise ArgumentError, "unknown settings validation mode"
47
+ end
48
+ backup_directory
49
+ return self if mode == :export
50
+
51
+ client_options
52
+ rules
53
+ concurrency
54
+ limit_per_vendor
55
+ log_directory
56
+ connection_options
57
+ Net::Connector.vendors.each { |vendor| connection_options(vendor) }
58
+ tftp_source_files.each_value { |source| TftpTarget.validate_source_file!(source) }
59
+ tftp_vrfs
60
+ TftpTarget.new(host: tftp_server, path: "preflight.cfg") if tftp_server
61
+ self
62
+ end
63
+
64
+ def client(policy: snapshot(mode: :inventory))
65
+ options = policy.client_options
66
+ raise ArgumentError, "NETDISCO_URL is required" unless options.fetch(:url)
67
+
68
+ authentication = if (token = secret_value("NETDISCO_API_KEY"))
69
+ { api_key: token }
70
+ else
71
+ { username: required_secret("NETDISCO_USERNAME"), password: required_secret("NETDISCO_PASSWORD") }
72
+ end
73
+ Client.new(**options, **authentication)
74
+ end
75
+
76
+ # 原生客户端和 --show-config 共享同一预算、地址与 HTTP 策略校验。
77
+ def client_options
78
+ options = Client::DEFAULTS.to_h do |name, default|
79
+ key = "NETDISCO_#{name.to_s.upcase}"
80
+ parsed = case name
81
+ when :inventory_timeout then number(key, default)
82
+ when :allow_insecure_http then boolean(key, default)
83
+ else integer(key, default)
84
+ end
85
+ [name, parsed]
86
+ end
87
+ options = Client.options(**options)
88
+ url = value("NETDISCO_URL")
89
+ Client.validate_url!(url, allow_insecure_http: options.fetch(:allow_insecure_http)) if url
90
+ options.merge(url: url).freeze
36
91
  end
37
92
 
38
- # 读取清单筛选和厂商映射规则。
39
93
  def rules
40
94
  Rules.new(include_hosts: list("NET_CONNECTOR_INCLUDE_HOSTS"),
41
95
  exclude_hosts: list("NET_CONNECTOR_EXCLUDE_HOSTS"),
@@ -45,15 +99,13 @@ module Net
45
99
  mappings: json("NET_CONNECTOR_DEVICE_RULES", []))
46
100
  end
47
101
 
48
- # 读取并展开本地备份目录。
49
102
  def backup_directory
50
- path = @env.fetch("NET_CONNECTOR_BACKUP_DIRECTORY", DEFAULT_BACKUP_DIRECTORY)
103
+ path = raw("NET_CONNECTOR_BACKUP_DIRECTORY", DEFAULT_BACKUP_DIRECTORY)
51
104
  raise ArgumentError, "NET_CONNECTOR_BACKUP_DIRECTORY must be nonempty" unless present?(path)
52
105
 
53
106
  File.expand_path(path)
54
107
  end
55
108
 
56
- # 读取并限制设备任务并发数。
57
109
  def concurrency
58
110
  count = integer("NET_CONNECTOR_CONCURRENCY", DEFAULT_CONCURRENCY)
59
111
  raise ArgumentError, "NET_CONNECTOR_CONCURRENCY must be at most #{MAX_CONCURRENCY}" if count > MAX_CONCURRENCY
@@ -61,121 +113,172 @@ module Net
61
113
  count
62
114
  end
63
115
 
64
- # 确定厂商采样数量:本地默认全量,TFTP 默认每厂商五台。
65
116
  def limit_per_vendor(tftp: false)
66
- return integer("NET_CONNECTOR_SAMPLE_PER_VENDOR", 5) if @env.key?("NET_CONNECTOR_SAMPLE_PER_VENDOR")
67
-
68
- 5 if tftp
117
+ count = if configured?("NET_CONNECTOR_SAMPLE_PER_VENDOR")
118
+ integer("NET_CONNECTOR_SAMPLE_PER_VENDOR", 5) unless raw("NET_CONNECTOR_SAMPLE_PER_VENDOR").nil?
119
+ elsif tftp
120
+ 5
121
+ end
122
+ Planner.validate_limit!(count)
123
+ count
69
124
  end
70
125
 
71
- # 读取厂商的 TFTP 源文件配置,并提供华为启动配置默认路径。
72
126
  def tftp_source_files
73
- files = { huawei: @env.fetch("NET_CONNECTOR_HUAWEI_TFTP_SOURCE_FILE", "flash:/startup.cfg") }
127
+ files = { huawei: raw("NET_CONNECTOR_HUAWEI_TFTP_SOURCE_FILE", "flash:/startup.cfg") }
74
128
  %i[h3c h3c_wireless].each do |vendor|
75
129
  key = "NET_CONNECTOR_#{vendor.to_s.upcase}_TFTP_SOURCE_FILE"
76
- files[vendor] = @env.fetch(key) if @env.key?(key)
130
+ files[vendor] = raw(key) if configured?(key)
77
131
  end
78
132
  files
79
133
  end
80
134
 
81
- # 读取各厂商 TFTP 出口 VRF 的名称映射。
82
135
  def tftp_vrfs
83
- raw = json("NET_CONNECTOR_TFTP_VRFS", {})
84
- raise ArgumentError, "NET_CONNECTOR_TFTP_VRFS must be a JSON object" unless raw.is_a?(Hash)
136
+ mapping = json("NET_CONNECTOR_TFTP_VRFS", {})
137
+ raise ArgumentError, "NET_CONNECTOR_TFTP_VRFS must be a JSON object" unless mapping.is_a?(Hash)
85
138
 
86
- raw.transform_keys(&:to_sym)
139
+ self.class.validate_vrfs!(mapping.transform_keys(&:to_sym))
140
+ end
141
+
142
+ def self.validate_vrfs!(vrfs)
143
+ unless vrfs.is_a?(Hash) && vrfs.all? { |vendor, name|
144
+ %i[cisco_nxos hillstone].include?(vendor) && name.is_a?(String) &&
145
+ name.match?(/\A[A-Za-z0-9_][A-Za-z0-9_.-]*\z/)
146
+ }
147
+ raise ArgumentError, "vrfs must map supported vendor names to safe VRF names"
148
+ end
149
+ vrfs
87
150
  end
88
151
 
89
- # 每台设备执行时读取凭据,使新批次可使用轮换后的环境变量。
90
- # 按厂商优先级读取设备账号与密码。
91
- def credentials_for(device)
152
+ def tftp_server = value("TFTP_HOST")
153
+
154
+ def connection_options(vendor = nil)
155
+ prefix = "NET_CONNECTOR_#{vendor.to_s.upcase}_"
156
+ configuration = Configuration.new(
157
+ protocol: (vendor && value(prefix + "PROTOCOL")) || value("NET_CONNECTOR_PROTOCOL") || "ssh",
158
+ log_level: value("NET_CONNECTOR_LOG_LEVEL") || "info", known_hosts: value("NET_CONNECTOR_KNOWN_HOSTS"),
159
+ host_key_policy: value("NET_CONNECTOR_HOST_KEY_POLICY") || "strict",
160
+ max_script_output_bytes: optional_integer("NET_CONNECTOR_MAX_SCRIPT_OUTPUT_BYTES")
161
+ )
162
+ options = %i[protocol log_level known_hosts host_key_policy].to_h { |key| [key, configuration.public_send(key)] }
163
+ options[:max_script_output_bytes] = configuration.max_script_output_bytes if configuration.max_script_output_bytes
164
+ options.freeze
165
+ end
166
+
167
+ def device_credentials_for(device)
92
168
  prefix = "NET_CONNECTOR_#{device.vendor.to_s.upcase}_"
93
- username = value(prefix + "USERNAME") || value("NET_CONNECTOR_DEVICE_USERNAME")
169
+ username = secret_value(prefix + "USERNAME") || secret_value("NET_CONNECTOR_DEVICE_USERNAME")
94
170
  return nil unless username
95
171
 
96
- {
97
- username: username,
98
- password: value(prefix + "PASSWORD") || value("NET_CONNECTOR_DEVICE_PASSWORD"),
99
- enable_password: value(prefix + "ENABLE_PASSWORD") || value("NET_CONNECTOR_ENABLE_PASSWORD"),
100
- protocol: (value(prefix + "PROTOCOL") || value("NET_CONNECTOR_PROTOCOL") || "ssh").to_sym,
101
- log_level: (value("NET_CONNECTOR_LOG_LEVEL") || "info").to_sym,
102
- known_hosts: value("NET_CONNECTOR_KNOWN_HOSTS"),
103
- host_key_policy: (value("NET_CONNECTOR_HOST_KEY_POLICY") || "strict").to_sym
104
- }
172
+ { username: username,
173
+ password: secret_value(prefix + "PASSWORD") || secret_value("NET_CONNECTOR_DEVICE_PASSWORD"),
174
+ enable_password: secret_value(prefix + "ENABLE_PASSWORD") || secret_value("NET_CONNECTOR_ENABLE_PASSWORD") }
105
175
  end
106
176
 
107
- # 读取并展开设备日志目录。
108
177
  def log_directory
109
178
  path = value("NET_CONNECTOR_LOG_DIRECTORY")
110
179
  File.expand_path(path) if path
111
180
  end
112
181
 
113
- # 仅展示非敏感配置,不输出设备凭据或 API 密钥。
114
- # 生成不包含凭据的可展示配置。
115
182
  def public_config
116
- rules
117
- url = value("NETDISCO_URL")
118
- if url
119
- uri = URI.parse(url)
120
- unless uri.is_a?(URI::HTTP) && uri.host && !uri.userinfo && !uri.query && !uri.fragment
121
- raise ArgumentError, "NETDISCO_URL must be a base URL without credentials or query"
122
- end
123
- end
183
+ snapshot(mode: :show_config).send(:config_hash)
184
+ end
185
+
186
+ def inspect = "#<#{self.class}>"
187
+
188
+ private
189
+
190
+ def config_hash
124
191
  {
125
- netdisco: { url: url, page_size: integer("NETDISCO_PAGE_SIZE", DEFAULT_PAGE_SIZE) },
126
- backup: { directory: backup_directory, concurrency: concurrency,
127
- limit_per_vendor: limit_per_vendor },
192
+ netdisco: client_options,
193
+ backup: { directory: backup_directory, concurrency: concurrency, limit_per_vendor: limit_per_vendor },
128
194
  inventory: { include_hosts: list("NET_CONNECTOR_INCLUDE_HOSTS"),
129
195
  exclude_hosts: list("NET_CONNECTOR_EXCLUDE_HOSTS"),
130
196
  include_vendors: list("NET_CONNECTOR_INCLUDE_VENDORS"),
131
197
  vendor_overrides: json("NET_CONNECTOR_VENDOR_OVERRIDES", {}),
132
198
  host_overrides: json("NET_CONNECTOR_HOST_OVERRIDES", {}),
133
199
  device_rules: json("NET_CONNECTOR_DEVICE_RULES", []) },
134
- ssh: { protocol: value("NET_CONNECTOR_PROTOCOL") || "ssh",
135
- known_hosts: value("NET_CONNECTOR_KNOWN_HOSTS"),
136
- host_key_policy: value("NET_CONNECTOR_HOST_KEY_POLICY") || "strict",
137
- log_directory: log_directory,
138
- log_level: value("NET_CONNECTOR_LOG_LEVEL") || "info" },
139
- tftp: { server: value("TFTP_HOST"), vrfs: tftp_vrfs.transform_keys(&:to_s) }
200
+ ssh: connection_options.transform_values { |item| item.is_a?(Symbol) ? item.to_s : item }.merge(
201
+ log_directory: log_directory,
202
+ vendor_protocols: Net::Connector.vendors.filter_map do |vendor|
203
+ protocol = value("NET_CONNECTOR_#{vendor.to_s.upcase}_PROTOCOL")
204
+ [vendor, protocol] if protocol
205
+ end.to_h
206
+ ),
207
+ tftp: { server: tftp_server, vrfs: tftp_vrfs.transform_keys(&:to_s), source_files: tftp_source_files }
140
208
  }
141
- rescue URI::InvalidURIError
142
- raise ArgumentError, "NETDISCO_URL is invalid"
143
209
  end
144
210
 
145
- private
211
+ def non_secret_keys
212
+ keys = ConfigFile::FIELDS.values.flat_map { |section| section.values.map(&:first) }
213
+ keys + Net::Connector.vendors.map { |vendor| "NET_CONNECTOR_#{vendor.to_s.upcase}_PROTOCOL" }
214
+ end
215
+
216
+ def raw(key, default = nil)
217
+ return @policy.send(:raw, key, default) if @policy
218
+ return @overrides.fetch(key) if @overrides.key?(key)
219
+ return @env.fetch(key) if @env.key?(key)
220
+
221
+ @defaults.fetch(key, default)
222
+ end
223
+
224
+ def configured?(key) = !raw(key, UNSET).equal?(UNSET)
146
225
 
147
- # 解析指定环境变量中的 JSON 配置。
148
226
  def json(key, default)
149
- raw = value(key)
150
- raw ? JSON.parse(raw) : default
227
+ text = value(key)
228
+ text ? JSON.parse(text) : default
151
229
  rescue JSON::ParserError
152
- raise ArgumentError, "#{key} must be valid JSON"
230
+ raise ArgumentError, "#{key} must be valid JSON", cause: nil
153
231
  end
154
232
 
155
- # 解析逗号分隔的环境变量列表。
156
- def list(key) = @env.fetch(key, "").split(",").map(&:strip).reject(&:empty?)
233
+ def list(key) = (value(key) || "").split(",").map(&:strip).reject(&:empty?)
157
234
 
158
- # 读取正整数配置并统一错误信息。
159
235
  def integer(key, default)
160
- raw = @env.fetch(key, default)
161
- value = Integer(raw.to_s, 10)
162
- raise ArgumentError, "#{key} must be a positive integer" unless value.positive?
236
+ result = Integer(raw(key, default).to_s, 10)
237
+ raise ArgumentError unless result.positive?
163
238
 
164
- value
239
+ result
165
240
  rescue ArgumentError, TypeError
166
- raise ArgumentError, "#{key} must be a positive integer"
241
+ raise ArgumentError, "#{key} must be a positive integer", cause: nil
242
+ end
243
+
244
+ def optional_integer(key)
245
+ integer(key, nil) unless raw(key).nil?
246
+ end
247
+
248
+ def number(key, default)
249
+ input = raw(key, default)
250
+ result = input.is_a?(String) ? Float(input) : input
251
+ unless result.is_a?(Numeric) && result.real? && result.finite? && result.positive? && result.to_f.finite?
252
+ raise ArgumentError
253
+ end
254
+ result
255
+ rescue ArgumentError, TypeError
256
+ raise ArgumentError, "#{key} must be a positive finite number", cause: nil
257
+ end
258
+
259
+ def boolean(key, default)
260
+ result = raw(key, default)
261
+ return true if result == true || result == "true"
262
+ return false if result == false || result == "false"
263
+
264
+ raise ArgumentError, "#{key} must be true or false"
167
265
  end
168
266
 
169
- # 读取非空字符串配置。
170
267
  def value(key)
171
- result = @env[key]
268
+ result = raw(key)
269
+ raise ArgumentError, "#{key} must be a String" unless result.nil? || result.is_a?(String)
270
+
172
271
  result if present?(result)
173
272
  end
174
273
 
175
- # 读取必填环境变量。
176
- def required(key) = value(key) || (raise ArgumentError, "#{key} is required")
274
+ def secret_value(key)
275
+ result = @env[key]
276
+ raise ArgumentError, "#{key} must be a String" unless result.nil? || result.is_a?(String)
277
+
278
+ result.dup.freeze if present?(result)
279
+ end
177
280
 
178
- # 判断环境变量是否为非空字符串。
281
+ def required_secret(key) = secret_value(key) || (raise ArgumentError, "#{key} is required")
179
282
  def present?(value) = value.is_a?(String) && !value.strip.empty?
180
283
  end
181
284
  end
@@ -31,25 +31,10 @@ module Net
31
31
  Thread.current.report_on_exception = false
32
32
  while (task = queue.pop)
33
33
  index, device = task
34
- started_at = Time.now.utc
35
- begin
36
- on_start&.call(device)
37
- rescue StandardError => error
38
- callback_errors << { host: device.host, error_type: error.class.name }.freeze
39
- end
40
- result = begin
41
- yield device
42
- rescue StandardError => error
43
- on_error.call(device, error)
44
- end
45
- result = result.with(started_at: started_at, finished_at: Time.now.utc)
34
+ result = run_task(device, on_error, on_start, callback_errors) { yield device }
46
35
  outcomes[index] = result
47
36
  Array(on_result).each do |callback|
48
- begin
49
- callback.call(result)
50
- rescue StandardError => error
51
- callback_errors << { host: device.host, error_type: error.class.name }.freeze
52
- end
37
+ notify(callback, result, host: device.host, errors: callback_errors)
53
38
  end
54
39
  end
55
40
  ensure
@@ -76,6 +61,30 @@ module Net
76
61
  end
77
62
  Array.new(callback_errors.size) { callback_errors.pop }.freeze
78
63
  end
64
+
65
+ private
66
+
67
+ # 普通设备故障转换为结果,中断仍交给批次统一停止其余线程。
68
+ def run_task(device, on_error, on_start, callback_errors)
69
+ started_at = Time.now.utc
70
+ started = monotonic
71
+ notify(on_start, device, host: device.host, errors: callback_errors) unless on_start.nil?
72
+ result = begin
73
+ yield
74
+ rescue StandardError => error
75
+ on_error.call(device, error)
76
+ end
77
+ result.with(started_at: started_at, finished_at: Time.now.utc, duration_ms: ((monotonic - started) * 1000).round)
78
+ end
79
+
80
+ def monotonic = Process.clock_gettime(Process::CLOCK_MONOTONIC)
81
+
82
+ # 回调故障不覆盖设备结果,也不把异常消息中的凭据写入报告。
83
+ def notify(callback, value, host:, errors:)
84
+ callback.call(value)
85
+ rescue StandardError => error
86
+ errors << { host: host, error_type: error.class.name }.freeze
87
+ end
79
88
  end
80
89
  end
81
90
  end
@@ -0,0 +1,116 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "digest"
4
+ require_relative "../engine/errors"
5
+
6
+ module Net
7
+ module Connector
8
+ class BackupBusy < Error; end
9
+
10
+ module Storage
11
+ # 路径锁在设备采集之前取得;锁文件长期保留,避免等待者落在已经删除的 inode 上。
12
+ class BackupLock
13
+ STORAGE_KEY = :net_connector_backup_delegation
14
+ private_constant :STORAGE_KEY
15
+
16
+ attr_reader :path
17
+
18
+ def self.lock_path(path)
19
+ destination = File.expand_path(path)
20
+ directory = File.realpath(File.dirname(destination))
21
+ # 锁名先折叠 Unicode 与大小写,避免大小写不敏感文件系统上的别名绕过互斥。
22
+ # 在区分大小写的文件系统上可能保守地合并两个锁,但不会改写目标文件名。
23
+ filename = File.basename(destination).b.force_encoding(Encoding::UTF_8)
24
+ filename = filename.valid_encoding? ? filename.unicode_normalize(:nfc).downcase(:fold).unicode_normalize(:nfc) : filename.b.downcase
25
+ File.join(directory, ".net-connector-#{Digest::SHA256.hexdigest(filename)}.lock")
26
+ end
27
+
28
+ def self.synchronize(path, **options, &block) = new(path, **options).synchronize(&block)
29
+
30
+ def initialize(path, timeout: 0, host: nil)
31
+ raise ArgumentError, "path must be a nonempty String" unless path.is_a?(String) && !path.empty?
32
+ unless timeout.is_a?(Numeric) && timeout.real? && timeout.finite? && timeout >= 0 && timeout.to_f.finite?
33
+ raise ArgumentError, "lock_timeout must be a nonnegative finite number"
34
+ end
35
+
36
+ @path = File.expand_path(path).freeze
37
+ @lock_path = self.class.lock_path(@path).freeze
38
+ @timeout, @host = timeout.to_f, host
39
+ end
40
+
41
+ def synchronize
42
+ # Fleet 只向下层备份借用一次;消费后清除,采集回调中的递归备份仍须竞争锁。
43
+ # Thread#[] 是 Fiber 局部存储;fork 后不能消费父进程的借用记录。
44
+ if Thread.current[STORAGE_KEY] == [@lock_path, Process.pid]
45
+ Thread.current[STORAGE_KEY] = nil
46
+ return yield self
47
+ end
48
+
49
+ file = nil
50
+ begin
51
+ file = open_lock
52
+ verify_lock!(file)
53
+ acquire(file)
54
+ verify_lock!(file)
55
+ @owner = [Process.pid, Fiber.current]
56
+ yield self
57
+ ensure
58
+ @owner = nil
59
+ file&.close
60
+ end
61
+ end
62
+
63
+ # 仅持有实际 flock 的同一 Fiber 可授权一次内层采集,避免把路径锁变成任意可重入锁。
64
+ def with_delegated_lock
65
+ raise ThreadError, "backup path delegation requires lock ownership" unless @owner == [Process.pid, Fiber.current]
66
+
67
+ previous = Thread.current[STORAGE_KEY]
68
+ begin
69
+ Thread.current[STORAGE_KEY] = [@lock_path, Process.pid]
70
+ yield
71
+ ensure
72
+ Thread.current[STORAGE_KEY] = previous
73
+ end
74
+ end
75
+
76
+ private
77
+
78
+ def open_lock
79
+ File.open(@lock_path, File::RDWR | File::CREAT | File::NOFOLLOW | File::NONBLOCK, 0o600)
80
+ rescue Errno::ELOOP, Errno::EISDIR, Errno::ENXIO
81
+ raise ArgumentError, "backup lock must be a private regular file", cause: nil
82
+ end
83
+
84
+ def verify_lock!(file)
85
+ stat = file.stat
86
+ unless stat.file? && stat.uid == Process.euid && stat.nlink == 1 && (stat.mode & 0o7777) == 0o600
87
+ raise ArgumentError, "backup lock must be an owned regular file with mode 0600 and one link"
88
+ end
89
+ current = File.lstat(@lock_path)
90
+ unless current.file? && current.dev == stat.dev && current.ino == stat.ino
91
+ raise ArgumentError, "backup lock path changed while acquiring ownership"
92
+ end
93
+ rescue Errno::ENOENT
94
+ raise ArgumentError, "backup lock path changed while acquiring ownership", cause: nil
95
+ end
96
+
97
+ def acquire(file)
98
+ deadline = monotonic + @timeout
99
+ loop do
100
+ acquired = file.flock(File::LOCK_EX | File::LOCK_NB)
101
+ return if acquired && (@timeout.zero? || monotonic < deadline)
102
+
103
+ remaining = deadline - monotonic
104
+ if remaining <= 0
105
+ raise BackupBusy.new("backup path already belongs to another operation", host: @host, phase: :backup), cause: nil
106
+ end
107
+ wait([remaining, 0.05].min)
108
+ end
109
+ end
110
+
111
+ def monotonic = Process.clock_gettime(Process::CLOCK_MONOTONIC)
112
+ def wait(seconds) = sleep(seconds)
113
+ end
114
+ end
115
+ end
116
+ end