netstack 0.1.1 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +22 -3
- data/{LICENSE.txt → LICENSE} +7 -6
- data/NOTICE +25 -0
- data/README.md +58 -20
- data/docs/acceptance.md +92 -0
- data/docs/backups.md +100 -0
- data/docs/configuration.md +198 -0
- data/docs/connector.md +187 -0
- data/docs/ddi.md +142 -0
- data/docs/deployment.md +47 -0
- data/docs/foundations.md +45 -0
- data/docs/implementation.md +68 -0
- data/docs/inventory.md +42 -0
- data/docs/monitoring.md +227 -0
- data/docs/motor_integration.md +96 -0
- data/docs/performance.md +39 -0
- data/docs/release.md +20 -0
- data/docs/vxlan.md +91 -0
- data/docs/wireless.md +81 -0
- data/lib/netstack/backups/artifact.rb +23 -0
- data/lib/netstack/backups/diff.rb +15 -0
- data/lib/netstack/backups/repository.rb +77 -0
- data/lib/netstack/backups/request.rb +78 -0
- data/lib/netstack/circuits/business_calendar.rb +51 -0
- data/lib/netstack/circuits/capacity_review.rb +191 -0
- data/lib/netstack/circuits/endpoint.rb +25 -0
- data/lib/netstack/circuits/monitoring_plan.rb +212 -0
- data/lib/netstack/configuration/artifact.rb +85 -0
- data/lib/netstack/configuration/artifact_resolver.rb +30 -0
- data/lib/netstack/configuration/bootstrap/payload.rb +67 -0
- data/lib/netstack/configuration/bootstrap/validator.rb +224 -0
- data/lib/netstack/configuration/catalog.rb +60 -0
- data/lib/netstack/configuration/generator.rb +87 -0
- data/lib/netstack/configuration/input_contract.rb +82 -0
- data/lib/netstack/configuration/payload.rb +168 -0
- data/lib/netstack/configuration/reader.rb +124 -0
- data/lib/netstack/configuration/support.rb +30 -0
- data/lib/netstack/configuration/system_configuration.rb +56 -0
- data/lib/netstack/configuration/template_manager.rb +174 -0
- data/lib/netstack/configuration/template_store.rb +40 -0
- data/lib/netstack/configuration/validator.rb +566 -0
- data/lib/netstack/connector/authentication.rb +77 -0
- data/lib/netstack/connector/authentication_error.rb +11 -0
- data/lib/netstack/connector/backup.rb +19 -0
- data/lib/netstack/connector/backup_busy.rb +11 -0
- data/lib/netstack/connector/backup_persistence_error.rb +32 -0
- data/lib/netstack/connector/command.rb +63 -0
- data/lib/netstack/connector/command_result.rb +24 -0
- data/lib/netstack/connector/command_timeout.rb +11 -0
- data/lib/netstack/connector/configuration.rb +138 -0
- data/lib/netstack/connector/connection_closed.rb +11 -0
- data/lib/netstack/connector/connection_error.rb +11 -0
- data/lib/netstack/connector/device.rb +284 -0
- data/lib/netstack/connector/device_error.rb +11 -0
- data/lib/netstack/connector/dialogue.rb +55 -0
- data/lib/netstack/connector/error.rb +44 -0
- data/lib/netstack/connector/error_metadata.rb +58 -0
- data/lib/netstack/connector/event.rb +24 -0
- data/lib/netstack/connector/execution.rb +110 -0
- data/lib/netstack/connector/interaction.rb +55 -0
- data/lib/netstack/connector/internal_error.rb +11 -0
- data/lib/netstack/connector/known_hosts.rb +128 -0
- data/lib/netstack/connector/local_backup.rb +63 -0
- data/lib/netstack/connector/log/event.rb +65 -0
- data/lib/netstack/connector/log/formatter.rb +18 -0
- data/lib/netstack/connector/log/messages.rb +54 -0
- data/lib/netstack/connector/log/redacting_writer.rb +33 -0
- data/lib/netstack/connector/log/transcript.rb +35 -0
- data/lib/netstack/connector/log.rb +262 -0
- data/lib/netstack/connector/log_error.rb +11 -0
- data/lib/netstack/connector/login_timeout.rb +11 -0
- data/lib/netstack/connector/output_limit_exceeded.rb +11 -0
- data/lib/netstack/connector/parsing_error.rb +11 -0
- data/lib/netstack/connector/profile/builder.rb +238 -0
- data/lib/netstack/connector/profile.rb +225 -0
- data/lib/netstack/connector/prompt_error.rb +11 -0
- data/lib/netstack/connector/recovery.rb +59 -0
- data/lib/netstack/connector/redactor.rb +100 -0
- data/lib/netstack/connector/response.rb +25 -0
- data/lib/netstack/connector/response_reader.rb +140 -0
- data/lib/netstack/connector/result.rb +50 -0
- data/lib/netstack/connector/running_config/rendered.rb +16 -0
- data/lib/netstack/connector/running_config/strategy.rb +28 -0
- data/lib/netstack/connector/running_config.rb +113 -0
- data/lib/netstack/connector/save_config.rb +23 -0
- data/lib/netstack/connector/script.rb +59 -0
- data/lib/netstack/connector/script_error.rb +11 -0
- data/lib/netstack/connector/script_output_limit_exceeded.rb +11 -0
- data/lib/netstack/connector/session.rb +414 -0
- data/lib/netstack/connector/session_busy.rb +11 -0
- data/lib/netstack/connector/storage/backup_lock.rb +117 -0
- data/lib/netstack/connector/storage/private_file.rb +111 -0
- data/lib/netstack/connector/storage/safe_file.rb +64 -0
- data/lib/netstack/connector/storage/saved_config.rb +76 -0
- data/lib/netstack/connector/terminal_renderer.rb +186 -0
- data/lib/netstack/connector/terminal_text.rb +32 -0
- data/lib/netstack/connector/textfsm.rb +71 -0
- data/lib/netstack/connector/tftp/file_upload.rb +53 -0
- data/lib/netstack/connector/tftp/strategy.rb +67 -0
- data/lib/netstack/connector/tftp.rb +116 -0
- data/lib/netstack/connector/tftp_completion_error.rb +36 -0
- data/lib/netstack/connector/tftp_receipt.rb +54 -0
- data/lib/netstack/connector/tftp_target.rb +65 -0
- data/lib/netstack/connector/topology/checkpoint_error.rb +16 -0
- data/lib/netstack/connector/topology/deployment_adapter.rb +163 -0
- data/lib/netstack/connector/topology/execution_result.rb +34 -0
- data/lib/netstack/connector/topology/immediate_strategy.rb +39 -0
- data/lib/netstack/connector/topology/interface_description.rb +46 -0
- data/lib/netstack/connector/topology/interface_name.rb +60 -0
- data/lib/netstack/connector/topology/plan.rb +105 -0
- data/lib/netstack/connector/topology/strategy.rb +76 -0
- data/lib/netstack/connector/topology.rb +255 -0
- data/lib/netstack/connector/transport_error.rb +11 -0
- data/lib/netstack/connector/transports.rb +201 -0
- data/lib/netstack/connector/underlying_error.rb +25 -0
- data/lib/netstack/connector/unsupported_operation.rb +11 -0
- data/lib/netstack/connector/vendors/cisco_ios/running_config.rb +22 -0
- data/lib/netstack/connector/vendors/cisco_ios/tftp_backup.rb +41 -0
- data/lib/netstack/connector/vendors/cisco_ios/topology.rb +54 -0
- data/lib/netstack/connector/vendors/cisco_ios.rb +44 -0
- data/lib/netstack/connector/vendors/cisco_nxos/running_config.rb +25 -0
- data/lib/netstack/connector/vendors/cisco_nxos/tftp_backup.rb +43 -0
- data/lib/netstack/connector/vendors/cisco_nxos.rb +51 -0
- data/lib/netstack/connector/vendors/h3c/tftp_backup.rb +34 -0
- data/lib/netstack/connector/vendors/h3c/topology.rb +72 -0
- data/lib/netstack/connector/vendors/h3c.rb +75 -0
- data/lib/netstack/connector/vendors/h3c_wireless.rb +13 -0
- data/lib/netstack/connector/vendors/hillstone/running_config.rb +16 -0
- data/lib/netstack/connector/vendors/hillstone/tftp_backup.rb +55 -0
- data/lib/netstack/connector/vendors/hillstone/topology.rb +64 -0
- data/lib/netstack/connector/vendors/hillstone.rb +39 -0
- data/lib/netstack/connector/vendors/huawei/tftp_backup.rb +26 -0
- data/lib/netstack/connector/vendors/huawei.rb +62 -0
- data/lib/netstack/connector/vendors/palo_alto/running_config.rb +77 -0
- data/lib/netstack/connector/vendors/palo_alto/tftp_backup.rb +49 -0
- data/lib/netstack/connector/vendors/palo_alto/topology.rb +77 -0
- data/lib/netstack/connector/vendors/palo_alto.rb +39 -0
- data/lib/netstack/connector/vendors/radware/running_config.rb +21 -0
- data/lib/netstack/connector/vendors/radware/tftp_backup.rb +49 -0
- data/lib/netstack/connector/vendors/radware/topology.rb +19 -0
- data/lib/netstack/connector/vendors/radware.rb +59 -0
- data/lib/netstack/connector/write_timeout.rb +11 -0
- data/lib/netstack/connector.rb +32 -0
- data/lib/netstack/ddi/applier.rb +107 -0
- data/lib/netstack/ddi/deployment.rb +137 -0
- data/lib/netstack/ddi/inputs.rb +61 -0
- data/lib/netstack/ddi/manager.rb +197 -0
- data/lib/netstack/ddi/plan.rb +99 -0
- data/lib/netstack/ddi/scope.rb +127 -0
- data/lib/netstack/ddi/settings.rb +44 -0
- data/lib/netstack/ddi/state.rb +166 -0
- data/lib/netstack/deployment/checkpoint_error.rb +14 -0
- data/lib/netstack/deployment/cli_adapter.rb +64 -0
- data/lib/netstack/deployment/executor.rb +131 -0
- data/lib/netstack/deployment/plan.rb +82 -0
- data/lib/netstack/deployment/receipt.rb +78 -0
- data/lib/netstack/deployment/recovery.rb +19 -0
- data/lib/netstack/deployment/step.rb +16 -0
- data/lib/netstack/dimension.rb +30 -0
- data/lib/netstack/error.rb +14 -0
- data/lib/netstack/input_error.rb +9 -0
- data/lib/netstack/integrations/bluecat/client.rb +84 -0
- data/lib/netstack/integrations/http/client.rb +146 -0
- data/lib/netstack/integrations/http/error.rb +10 -0
- data/lib/netstack/integrations/http/response.rb +28 -0
- data/lib/netstack/integrations/http_delivery/adapter.rb +51 -0
- data/lib/netstack/integrations/http_delivery/client.rb +127 -0
- data/lib/netstack/integrations/http_delivery/result.rb +27 -0
- data/lib/netstack/integrations/oxidized/client.rb +89 -0
- data/lib/netstack/integrations/zabbix/client.rb +211 -0
- data/lib/netstack/integrations/zabbix/error.rb +10 -0
- data/lib/netstack/inventory/address.rb +30 -0
- data/lib/netstack/inventory/changes.rb +13 -0
- data/lib/netstack/inventory/cli_parser.rb +75 -0
- data/lib/netstack/inventory/collector.rb +99 -0
- data/lib/netstack/inventory/device.rb +14 -0
- data/lib/netstack/inventory/interface.rb +25 -0
- data/lib/netstack/inventory/neighbor.rb +15 -0
- data/lib/netstack/inventory/reconciler.rb +74 -0
- data/lib/netstack/inventory/snapshot.rb +69 -0
- data/lib/netstack/inventory/transceiver.rb +23 -0
- data/lib/netstack/ipv4.rb +204 -0
- data/lib/netstack/ipv6.rb +88 -0
- data/lib/netstack/monitoring/applier.rb +179 -0
- data/lib/netstack/monitoring/device_plan.rb +23 -0
- data/lib/netstack/monitoring/host_plan.rb +152 -0
- data/lib/netstack/monitoring/inputs.rb +73 -0
- data/lib/netstack/monitoring/measurements.rb +89 -0
- data/lib/netstack/monitoring/mutation.rb +37 -0
- data/lib/netstack/monitoring/plan.rb +69 -0
- data/lib/netstack/monitoring/probe_plan.rb +19 -0
- data/lib/netstack/monitoring/state_comparison.rb +37 -0
- data/lib/netstack/range_set.rb +181 -0
- data/lib/netstack/routing/lookup.rb +46 -0
- data/lib/netstack/routing/lookup_result.rb +14 -0
- data/lib/netstack/routing/route.rb +32 -0
- data/lib/netstack/routing/snapshot.rb +23 -0
- data/lib/netstack/target.rb +15 -0
- data/lib/netstack/unsupported_capability.rb +9 -0
- data/lib/netstack/values.rb +70 -0
- data/lib/netstack/version.rb +1 -1
- data/lib/netstack/vxlan/auditor.rb +187 -0
- data/lib/netstack/vxlan/cisco_nxos/config_parser.rb +327 -0
- data/lib/netstack/vxlan/cisco_nxos/deployment_adapter.rb +63 -0
- data/lib/netstack/vxlan/cisco_nxos/runtime_parser.rb +182 -0
- data/lib/netstack/vxlan/cisco_nxos/segment_renderer.rb +42 -0
- data/lib/netstack/vxlan/collector.rb +77 -0
- data/lib/netstack/vxlan/fabric.rb +29 -0
- data/lib/netstack/vxlan/finding.rb +20 -0
- data/lib/netstack/vxlan/node.rb +51 -0
- data/lib/netstack/vxlan/route_policy.rb +57 -0
- data/lib/netstack/vxlan/runtime_result.rb +24 -0
- data/lib/netstack/vxlan/segment.rb +35 -0
- data/lib/netstack/vxlan/segment_plan.rb +25 -0
- data/lib/netstack/vxlan/segment_planner.rb +156 -0
- data/lib/netstack/vxlan/snapshot.rb +34 -0
- data/lib/netstack/vxlan/validation.rb +58 -0
- data/lib/netstack/vxlan/verifier.rb +52 -0
- data/lib/netstack/vxlan/vpc_domain.rb +20 -0
- data/lib/netstack/wireless/access_point.rb +12 -0
- data/lib/netstack/wireless/ap_group.rb +12 -0
- data/lib/netstack/wireless/assessment.rb +13 -0
- data/lib/netstack/wireless/assessor.rb +83 -0
- data/lib/netstack/wireless/authentication_connection.rb +12 -0
- data/lib/netstack/wireless/bss.rb +12 -0
- data/lib/netstack/wireless/client.rb +12 -0
- data/lib/netstack/wireless/collector.rb +191 -0
- data/lib/netstack/wireless/configuration.rb +13 -0
- data/lib/netstack/wireless/finding.rb +12 -0
- data/lib/netstack/wireless/h3c/config_parser.rb +229 -0
- data/lib/netstack/wireless/h3c/lldp_parser.rb +69 -0
- data/lib/netstack/wireless/h3c/runtime_parser.rb +240 -0
- data/lib/netstack/wireless/huawei/runtime_parser.rb +83 -0
- data/lib/netstack/wireless/invalid_snapshot.rb +11 -0
- data/lib/netstack/wireless/neighbor.rb +12 -0
- data/lib/netstack/wireless/normalizer.rb +90 -0
- data/lib/netstack/wireless/parse_result.rb +13 -0
- data/lib/netstack/wireless/radio.rb +12 -0
- data/lib/netstack/wireless/reconciler.rb +115 -0
- data/lib/netstack/wireless/reconciliation.rb +11 -0
- data/lib/netstack/wireless/security_policy.rb +12 -0
- data/lib/netstack/wireless/service_template.rb +12 -0
- data/lib/netstack/wireless/service_template_bindings.rb +56 -0
- data/lib/netstack/wireless/snapshot.rb +150 -0
- data/lib/netstack/wireless/state.rb +23 -0
- data/lib/netstack/wireless/view.rb +154 -0
- data/lib/netstack.rb +10 -3
- data/resources/configuration/cisco-bootstrap.erb +118 -0
- data/resources/configuration/cisco-distributed-gateway.erb +40 -0
- data/resources/configuration/cisco-n9k-vpc.erb +62 -0
- data/resources/configuration/cisco-server-port.erb +43 -0
- data/resources/configuration/h3c-bootstrap.erb +108 -0
- data/resources/configuration/h3c-irf.erb +57 -0
- data/resources/configuration/h3c-server-port.erb +36 -0
- data/resources/configuration/hillstone-ipsec-vpn.erb +68 -0
- data/resources/configuration/hillstone-policy.erb +31 -0
- data/resources/configuration/paloalto-ipsec-vpn.erb +39 -0
- data/resources/configuration/radware-load-balancer.erb +26 -0
- data/resources/textfsm/cisco_cdp_neighbors_detail.textfsm +7 -0
- data/resources/textfsm/cisco_ios_running_config_interfaces.textfsm +12 -0
- data/resources/textfsm/cisco_ios_show_ip_interface_brief.textfsm +7 -0
- data/resources/textfsm/h3c_interface_descriptions.textfsm +11 -0
- data/resources/textfsm/h3c_lldp_local_first.textfsm +8 -0
- data/resources/textfsm/h3c_lldp_name_first.textfsm +8 -0
- data/resources/textfsm/hillstone_interface_descriptions.textfsm +11 -0
- data/resources/textfsm/hillstone_lldp_neighbors.textfsm +7 -0
- data/resources/textfsm/index +8 -0
- data/resources/textfsm/palo_alto_interface_descriptions.textfsm +6 -0
- data/resources/textfsm/palo_alto_lldp_neighbors.textfsm +11 -0
- data/resources/textfsm/radware_port_names.textfsm +13 -0
- metadata +588 -27
- data/.rubocop.yml +0 -13
- data/CODE_OF_CONDUCT.md +0 -84
- data/Gemfile +0 -10
- data/Gemfile.lock +0 -40
- data/Rakefile +0 -8
- data/bin/console +0 -15
- data/bin/setup +0 -8
|
@@ -0,0 +1,77 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "ipaddr"
|
|
4
|
+
|
|
5
|
+
module Netstack
|
|
6
|
+
# 外部备份身份与证据;访问控制和保留策略由宿主管理。
|
|
7
|
+
module Backups
|
|
8
|
+
# Device association and version ownership belong here; storage and permissions do not.
|
|
9
|
+
class Repository
|
|
10
|
+
attr_reader :client
|
|
11
|
+
|
|
12
|
+
def initialize(client:)
|
|
13
|
+
@client = client
|
|
14
|
+
end
|
|
15
|
+
|
|
16
|
+
def node(target:, aliases: [], nodes: client.nodes)
|
|
17
|
+
raise InputError, "Backup aliases must be an array" unless aliases.is_a?(Array)
|
|
18
|
+
candidates = [target.key, target.host, *aliases].map { |value| Values.text(value, name: "node alias").downcase }.uniq
|
|
19
|
+
host = canonical_host(target.host)
|
|
20
|
+
matches = nodes.select do |row|
|
|
21
|
+
values = row.values_at("name", "full_name", "ip").compact.map { |value| Values.text(value, name: "node identity").downcase }
|
|
22
|
+
!(values & candidates).empty? || (row["ip"] && canonical_host(row["ip"]) == host)
|
|
23
|
+
end
|
|
24
|
+
raise Netstack::Error.new("Multiple backup nodes match the target", code: :ambiguous_node) if matches.length > 1
|
|
25
|
+
match = matches.first
|
|
26
|
+
if match && match["ip"] && canonical_host(match["ip"]) != host
|
|
27
|
+
raise Netstack::Error.new("Backup node address conflicts with target", code: :node_identity_conflict)
|
|
28
|
+
end
|
|
29
|
+
match && Values.copy(match)
|
|
30
|
+
end
|
|
31
|
+
|
|
32
|
+
def node!(target:, aliases: [])
|
|
33
|
+
node(target: target, aliases: aliases) || raise(Netstack::Error.new("Backup node was not found", code: :node_not_found))
|
|
34
|
+
end
|
|
35
|
+
|
|
36
|
+
def configuration(node:)
|
|
37
|
+
Artifact.new(client.configuration(**identity(node)))
|
|
38
|
+
end
|
|
39
|
+
|
|
40
|
+
def versions(node:)
|
|
41
|
+
client.versions(**identity(node))
|
|
42
|
+
end
|
|
43
|
+
|
|
44
|
+
def diff(node:, from_oid:, to_oid:)
|
|
45
|
+
known = versions(node: node).map { |row| row.fetch("oid") }
|
|
46
|
+
unless from_oid != to_oid && known.include?(from_oid) && known.include?(to_oid)
|
|
47
|
+
raise InputError, "Select two distinct versions belonging to this node"
|
|
48
|
+
end
|
|
49
|
+
Diff.new(client.diff(**identity(node), from_oid: from_oid, to_oid: to_oid))
|
|
50
|
+
end
|
|
51
|
+
|
|
52
|
+
def identity(node)
|
|
53
|
+
{ node_name: Values.text(node.fetch("name"), name: "node name"), group: Values.text(node["group"], name: "node group", optional: true) }
|
|
54
|
+
end
|
|
55
|
+
|
|
56
|
+
# Stable device identity available from Oxidized; status and timestamps are excluded.
|
|
57
|
+
def fingerprint(node)
|
|
58
|
+
Values.copy(node.slice("name", "group", "ip", "full_name", "model"))
|
|
59
|
+
end
|
|
60
|
+
|
|
61
|
+
def target_fingerprint(target)
|
|
62
|
+
serial = target.attributes[:serial] || target.attributes["serial"]
|
|
63
|
+
Values.copy({ "key" => target.key, "host" => target.host, "platform" => target.platform,
|
|
64
|
+
"attributes" => { "serial" => serial }.compact })
|
|
65
|
+
end
|
|
66
|
+
|
|
67
|
+
private
|
|
68
|
+
|
|
69
|
+
def canonical_host(value)
|
|
70
|
+
text = Values.text(value, name: "node address").downcase.delete_suffix(".")
|
|
71
|
+
IPAddr.new(text).to_s
|
|
72
|
+
rescue IPAddr::InvalidAddressError
|
|
73
|
+
text
|
|
74
|
+
end
|
|
75
|
+
end
|
|
76
|
+
end
|
|
77
|
+
end
|
|
@@ -0,0 +1,78 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "time"
|
|
4
|
+
|
|
5
|
+
module Netstack
|
|
6
|
+
module Backups
|
|
7
|
+
# A successful queue response means accepted, never completed. A host persists
|
|
8
|
+
# each returned event under execution_id and invokes recover to poll again.
|
|
9
|
+
class Request
|
|
10
|
+
def initialize(repository:, clock: -> { Time.now.utc })
|
|
11
|
+
@repository = repository
|
|
12
|
+
@clock = clock
|
|
13
|
+
end
|
|
14
|
+
|
|
15
|
+
def call(target:, execution_id:, checkpoint:, aliases: [])
|
|
16
|
+
raise InputError, "checkpoint must be callable" unless checkpoint.respond_to?(:call)
|
|
17
|
+
id = Values.text(execution_id, name: "execution_id")
|
|
18
|
+
nodes = @repository.client.nodes
|
|
19
|
+
node = @repository.node(target: target, aliases: aliases, nodes: nodes)
|
|
20
|
+
raise Netstack::Error.new("Backup node was not found", code: :node_not_found) unless node
|
|
21
|
+
# Upstream ignores group in /node/next, so group alone cannot disambiguate a queue write.
|
|
22
|
+
if nodes.count { |row| row["name"] == node["name"] } != 1
|
|
23
|
+
raise Netstack::Error.new("Oxidized queue requires a globally unique node name", code: :ambiguous_node)
|
|
24
|
+
end
|
|
25
|
+
event = { "execution_id" => id, "node" => node.slice("name", "group"),
|
|
26
|
+
"node_fingerprint" => @repository.fingerprint(node), "target" => @repository.target_fingerprint(target),
|
|
27
|
+
"requested_at" => @clock.call.utc.iso8601(6), "baseline_end" => node.dig("last", "end"), "stage" => "started" }
|
|
28
|
+
persist!(checkpoint, event)
|
|
29
|
+
@repository.client.request_backup(**@repository.identity(node))
|
|
30
|
+
event = event.merge("stage" => "receipt")
|
|
31
|
+
persist!(checkpoint, event)
|
|
32
|
+
Values.copy(event)
|
|
33
|
+
end
|
|
34
|
+
|
|
35
|
+
def recover(execution_id:, events:, checkpoint:)
|
|
36
|
+
id = Values.text(execution_id, name: "execution_id")
|
|
37
|
+
raise InputError, "checkpoint must be callable" unless checkpoint.respond_to?(:call)
|
|
38
|
+
unless events.is_a?(Array) && !events.empty? && events.all? { |event| event.is_a?(Hash) } && events.map { |event| event["stage"] }.then { |stages| [%w[started], %w[started receipt], %w[started receipt verified], %w[started verified]].include?(stages) }
|
|
39
|
+
raise InputError, "Invalid backup checkpoint sequence"
|
|
40
|
+
end
|
|
41
|
+
first = events.first
|
|
42
|
+
unless events.all? { |event| event.is_a?(Hash) && event["execution_id"] == id && event.except("stage", "completed_at") == first.except("stage", "completed_at") }
|
|
43
|
+
raise InputError, "Backup checkpoints do not belong to this execution"
|
|
44
|
+
end
|
|
45
|
+
return Values.copy(events.last) if events.last["stage"] == "verified"
|
|
46
|
+
|
|
47
|
+
requested = Values.time(first.fetch("requested_at"), name: "requested_at")
|
|
48
|
+
identity = first.fetch("node")
|
|
49
|
+
nodes = @repository.client.nodes.select { |row| row.slice("name", "group") == identity }
|
|
50
|
+
raise Netstack::Error.new("Backup node identity changed", code: :unknown_result) unless nodes.length == 1
|
|
51
|
+
unless first["node_fingerprint"].is_a?(Hash) && @repository.fingerprint(nodes.first) == first["node_fingerprint"]
|
|
52
|
+
raise Netstack::Error.new("Backup node identity changed", code: :unknown_result)
|
|
53
|
+
end
|
|
54
|
+
last = nodes.first["last"]
|
|
55
|
+
return Values.copy(first.merge("stage" => "pending")) unless last.is_a?(Hash) && last["start"] && last["end"]
|
|
56
|
+
started = Values.time(last["start"], name: "backup start")
|
|
57
|
+
ended = Values.time(last["end"], name: "backup end")
|
|
58
|
+
return Values.copy(first.merge("stage" => "pending")) unless started >= requested && ended >= started && last["end"] != first["baseline_end"]
|
|
59
|
+
unless %w[success ok].include?(last["status"])
|
|
60
|
+
raise Netstack::Error.new("Backup attempt failed", code: :backup_failed, stage: :verification)
|
|
61
|
+
end
|
|
62
|
+
result = first.merge("stage" => "verified", "completed_at" => ended.iso8601(6))
|
|
63
|
+
persist!(checkpoint, result)
|
|
64
|
+
Values.copy(result)
|
|
65
|
+
end
|
|
66
|
+
|
|
67
|
+
private
|
|
68
|
+
|
|
69
|
+
def persist!(checkpoint, event)
|
|
70
|
+
raise Netstack::Error.new("Backup checkpoint was not committed", code: :checkpoint_failed, stage: :checkpoint) unless checkpoint.call(Values.copy(event)) == true
|
|
71
|
+
rescue Netstack::Error
|
|
72
|
+
raise
|
|
73
|
+
rescue StandardError
|
|
74
|
+
raise Netstack::Error.new("Backup checkpoint failed", code: :checkpoint_failed, stage: :checkpoint), cause: nil
|
|
75
|
+
end
|
|
76
|
+
end
|
|
77
|
+
end
|
|
78
|
+
end
|
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "date"
|
|
4
|
+
|
|
5
|
+
module Netstack
|
|
6
|
+
module Circuits
|
|
7
|
+
# A fixed-offset business calendar. For DST zones, inject a callable calendar
|
|
8
|
+
# using the host's timezone library instead of changing ENV['TZ'] globally.
|
|
9
|
+
class BusinessCalendar
|
|
10
|
+
attr_reader :daily_seconds
|
|
11
|
+
|
|
12
|
+
def initialize(utc_offset:, weekdays: [1, 2, 3, 4, 5], starts_at: "09:00", ends_at: "18:00", holidays: [])
|
|
13
|
+
valid_weekdays = weekdays.is_a?(Array) && weekdays.any? && weekdays.all? { |day| day.is_a?(Integer) && (0..6).cover?(day) }
|
|
14
|
+
unless utc_offset.is_a?(String) && utc_offset.match?(/\A[+-](?:0\d|1[0-4]):[0-5]\d\z/) && valid_weekdays
|
|
15
|
+
Monitoring::Inputs.invalid!("Invalid business calendar")
|
|
16
|
+
end
|
|
17
|
+
@utc_offset = utc_offset.freeze
|
|
18
|
+
@weekdays = weekdays.uniq.freeze
|
|
19
|
+
@start = minutes(starts_at)
|
|
20
|
+
@finish = minutes(ends_at)
|
|
21
|
+
Monitoring::Inputs.invalid!("Business hours must finish on the same day after they start") unless @finish > @start
|
|
22
|
+
@daily_seconds = (@finish - @start) * 60
|
|
23
|
+
@holidays = holidays.map { |day| Date.iso8601(day).iso8601 }.freeze
|
|
24
|
+
rescue Date::Error, TypeError
|
|
25
|
+
Monitoring::Inputs.invalid!("Invalid business holiday")
|
|
26
|
+
end
|
|
27
|
+
|
|
28
|
+
def call(time)
|
|
29
|
+
local = time.getlocal(@utc_offset)
|
|
30
|
+
date = local.to_date.iso8601
|
|
31
|
+
workday = @weekdays.include?(local.wday) && !@holidays.include?(date)
|
|
32
|
+
minute = (local.hour * 60) + local.min
|
|
33
|
+
{ "date" => date, "business" => workday && minute >= @start && minute < @finish }
|
|
34
|
+
end
|
|
35
|
+
|
|
36
|
+
def consecutive_workdays?(previous, current)
|
|
37
|
+
date = Date.iso8601(previous) + 1
|
|
38
|
+
date += 1 until @weekdays.include?(date.wday) && !@holidays.include?(date.iso8601)
|
|
39
|
+
date.iso8601 == current
|
|
40
|
+
end
|
|
41
|
+
|
|
42
|
+
private
|
|
43
|
+
|
|
44
|
+
def minutes(value)
|
|
45
|
+
Monitoring::Inputs.invalid!("Business time must be HH:MM") unless value.is_a?(String) && value.match?(/\A(?:[01]\d|2[0-3]):[0-5]\d\z/)
|
|
46
|
+
hour, minute = value.split(":").map(&:to_i)
|
|
47
|
+
(hour * 60) + minute
|
|
48
|
+
end
|
|
49
|
+
end
|
|
50
|
+
end
|
|
51
|
+
end
|
|
@@ -0,0 +1,191 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Netstack
|
|
4
|
+
# 链路监控与容量分析;调度和通知由宿主负责。
|
|
5
|
+
module Circuits
|
|
6
|
+
# 先评估覆盖率再计算利用率,避免将缺失样本误判为
|
|
7
|
+
# 空闲容量;注入的日历定义业务时段与工作日。
|
|
8
|
+
class CapacityReview
|
|
9
|
+
WINDOW_SECONDS = { "7" => 7 * 86_400, "15" => 15 * 86_400, "31" => 31 * 86_400 }.freeze
|
|
10
|
+
|
|
11
|
+
def initialize(calendar:, interval_seconds: 300, minimum_coverage: 0.9, minimum_samples: 12)
|
|
12
|
+
unless calendar.respond_to?(:call) && calendar.respond_to?(:daily_seconds) && calendar.respond_to?(:consecutive_workdays?) &&
|
|
13
|
+
interval_seconds.is_a?(Integer) && interval_seconds.positive? && minimum_samples.is_a?(Integer) && minimum_samples.positive?
|
|
14
|
+
Monitoring::Inputs.invalid!("Invalid capacity review sampling policy")
|
|
15
|
+
end
|
|
16
|
+
@calendar = calendar
|
|
17
|
+
@interval = interval_seconds
|
|
18
|
+
@minimum_coverage = Monitoring::Inputs.number(minimum_coverage, positive: true)
|
|
19
|
+
Monitoring::Inputs.invalid!("Coverage must not exceed one") if @minimum_coverage > 1
|
|
20
|
+
@minimum_samples = minimum_samples
|
|
21
|
+
end
|
|
22
|
+
|
|
23
|
+
def summarize(samples:, inbound_capacity_mbps:, outbound_capacity_mbps:, from:, to:, complete: true)
|
|
24
|
+
Monitoring::Inputs.boolean(complete)
|
|
25
|
+
unless from.is_a?(Integer) && from >= 0 && to.is_a?(Integer) && to >= from && (((to - from) / @interval) + 1) <= 2_000_000
|
|
26
|
+
Monitoring::Inputs.invalid!("Invalid capacity review interval")
|
|
27
|
+
end
|
|
28
|
+
capacities = [inbound_capacity_mbps, outbound_capacity_mbps].map { |value| Monitoring::Inputs.number(value, positive: true) * 1_000_000 }
|
|
29
|
+
values = samples_by_slot(samples, from, to)
|
|
30
|
+
business = []
|
|
31
|
+
off = []
|
|
32
|
+
days = {}
|
|
33
|
+
(from..to).step(@interval) do |clock|
|
|
34
|
+
schedule = @calendar.call(Time.at(clock).utc)
|
|
35
|
+
sample = values[(clock - from) / @interval] || [nil, nil]
|
|
36
|
+
if schedule.fetch("business")
|
|
37
|
+
business << [clock, *sample]
|
|
38
|
+
day = (days[schedule.fetch("date")] ||= [])
|
|
39
|
+
day << sample
|
|
40
|
+
else
|
|
41
|
+
off << sample
|
|
42
|
+
end
|
|
43
|
+
end
|
|
44
|
+
counts = (0..1).map { |direction| business.count { |row| !row[direction + 1].nil? } }
|
|
45
|
+
coverage = counts.map { |count| business.empty? ? 0.0 : count.fdiv(business.size) }
|
|
46
|
+
sufficient = complete && counts.all? { |count| count >= @minimum_samples } && coverage.all? { |value| value >= @minimum_coverage }
|
|
47
|
+
stats = {
|
|
48
|
+
"from" => from, "to" => to, "interval_seconds" => @interval,
|
|
49
|
+
"status" => sufficient ? "ok" : "insufficient_data", "expected_samples" => business.size,
|
|
50
|
+
"in_samples" => counts[0], "out_samples" => counts[1], "in_coverage" => coverage[0], "out_coverage" => coverage[1],
|
|
51
|
+
"high_continuous_minutes" => high_minutes(business, capacities),
|
|
52
|
+
"low_continuous_workdays" => low_days(days, capacities)
|
|
53
|
+
}
|
|
54
|
+
%w[in out].each_with_index do |direction, index|
|
|
55
|
+
traffic = business.map { |row| row[index + 1] }.compact
|
|
56
|
+
p95 = percentile(traffic)
|
|
57
|
+
maximum = traffic.max
|
|
58
|
+
off_maximum = off.map { |row| row[index] }.compact.max
|
|
59
|
+
stats.merge!("#{direction}_p95_bps" => p95, "#{direction}_max_bps" => maximum,
|
|
60
|
+
"#{direction}_p95_utilization" => p95 && (p95 / capacities[index]),
|
|
61
|
+
"#{direction}_max_utilization" => maximum && (maximum / capacities[index]),
|
|
62
|
+
"off_#{direction}_max_utilization" => off_maximum && (off_maximum / capacities[index]))
|
|
63
|
+
end
|
|
64
|
+
Values.copy(stats)
|
|
65
|
+
end
|
|
66
|
+
|
|
67
|
+
def decide(windows:)
|
|
68
|
+
windows = Monitoring::Inputs.object(windows)
|
|
69
|
+
windows = windows.select { |days, summary| valid_window?(days, summary) }
|
|
70
|
+
valid = windows.values
|
|
71
|
+
return decision("insufficient_data", "insufficient_business_samples") if valid.empty?
|
|
72
|
+
return decision("insufficient_data", "inconsistent_window_end") unless valid.map { |summary| summary["to"] }.uniq.one?
|
|
73
|
+
|
|
74
|
+
recent = windows["7"]
|
|
75
|
+
notices = if valid.any? do |summary|
|
|
76
|
+
%w[off_in_max_utilization off_out_max_utilization].any? do |key|
|
|
77
|
+
summary[key] && summary[key] >= 0.9
|
|
78
|
+
end
|
|
79
|
+
end
|
|
80
|
+
["off_hours_peak"]
|
|
81
|
+
else
|
|
82
|
+
[]
|
|
83
|
+
end
|
|
84
|
+
return decision("expand_now", "recent_p95_at_capacity", notices) if recent && recent["status"] == "ok" && maximum_utilization(recent) >= 0.95
|
|
85
|
+
return decision("expand_review", "sustained_high_utilization", notices) if valid.any? { |summary| summary.fetch("high_continuous_minutes", 0) >= 60 }
|
|
86
|
+
return decision("expand_review", "multiple_high_windows", notices) if valid.count { |summary| maximum_utilization(summary) >= 0.9 } >= 2
|
|
87
|
+
if valid.any? { |summary| summary.fetch("low_continuous_workdays", 0) >= 10 } || %w[15 31].all? { |key| low_window?(windows[key], 0.2) }
|
|
88
|
+
return decision("shrink_review", "sustained_low_utilization", notices)
|
|
89
|
+
end
|
|
90
|
+
|
|
91
|
+
decision("watch", "review_threshold_not_reached", notices)
|
|
92
|
+
end
|
|
93
|
+
|
|
94
|
+
private
|
|
95
|
+
|
|
96
|
+
def valid_window?(days, summary)
|
|
97
|
+
duration = WINDOW_SECONDS[days]
|
|
98
|
+
return false unless duration && summary.is_a?(Hash) && summary["status"] == "ok"
|
|
99
|
+
first, last, interval = summary.values_at("from", "to", "interval_seconds")
|
|
100
|
+
return false unless first.is_a?(Integer) && first >= 0 && last.is_a?(Integer) && last - first == duration && interval.is_a?(Integer) && interval.positive?
|
|
101
|
+
required = %w[in_p95_utilization out_p95_utilization high_continuous_minutes low_continuous_workdays]
|
|
102
|
+
required.all? { |key| summary[key].is_a?(Numeric) && summary[key].finite? && summary[key] >= 0 } &&
|
|
103
|
+
%w[off_in_max_utilization off_out_max_utilization].all? { |key| summary[key].nil? || (summary[key].is_a?(Numeric) && summary[key].finite? && summary[key] >= 0) }
|
|
104
|
+
end
|
|
105
|
+
|
|
106
|
+
def samples_by_slot(samples, from, to)
|
|
107
|
+
Monitoring::Inputs.invalid!("Capacity samples must be an array") unless samples.is_a?(Array)
|
|
108
|
+
slots = {}
|
|
109
|
+
seen = {}
|
|
110
|
+
ordered = samples.map do |sample|
|
|
111
|
+
sample = Monitoring::Inputs.object(sample)
|
|
112
|
+
Monitoring::Inputs.invalid!("Invalid capacity sample timestamp") unless sample["clock"].is_a?(Integer)
|
|
113
|
+
sample["ns"] = sample.fetch("ns", 0)
|
|
114
|
+
Monitoring::Inputs.invalid!("Invalid capacity sample nanoseconds") unless sample["ns"].is_a?(Integer) && (0...1_000_000_000).cover?(sample["ns"])
|
|
115
|
+
sample
|
|
116
|
+
end.sort_by { |sample| [sample.fetch("clock"), sample.fetch("ns")] }
|
|
117
|
+
ordered.each do |sample|
|
|
118
|
+
clock = sample.fetch("clock")
|
|
119
|
+
identity = [clock, sample.fetch("ns")]
|
|
120
|
+
Monitoring::Inputs.invalid!("Invalid or duplicate capacity sample timestamp") unless clock >= from && clock <= to && !seen[identity]
|
|
121
|
+
seen[identity] = true
|
|
122
|
+
slot = (clock - from) / @interval
|
|
123
|
+
values = %w[in_bps out_bps].map { |key| sample[key].nil? ? nil : Monitoring::Inputs.number(sample[key]) }
|
|
124
|
+
# Multiple observations in a sampling interval use the latest value per
|
|
125
|
+
# direction, without inventing a value for an unobserved direction.
|
|
126
|
+
previous = slots.fetch(slot, [nil, nil])
|
|
127
|
+
slots[slot] = values.each_with_index.map { |value, index| value.nil? ? previous[index] : value }
|
|
128
|
+
end
|
|
129
|
+
slots
|
|
130
|
+
rescue KeyError
|
|
131
|
+
Monitoring::Inputs.invalid!("Capacity sample lacks a timestamp")
|
|
132
|
+
end
|
|
133
|
+
|
|
134
|
+
def percentile(values)
|
|
135
|
+
return nil if values.empty?
|
|
136
|
+
|
|
137
|
+
values.sort.fetch((values.size * 0.95).ceil - 1)
|
|
138
|
+
end
|
|
139
|
+
|
|
140
|
+
def high_minutes(rows, capacities)
|
|
141
|
+
start = nil
|
|
142
|
+
previous_clock = nil
|
|
143
|
+
maximum = 0
|
|
144
|
+
rows.each do |clock, inbound, outbound|
|
|
145
|
+
high = !inbound.nil? && !outbound.nil? && [inbound / capacities[0], outbound / capacities[1]].max >= 0.9
|
|
146
|
+
if high
|
|
147
|
+
start = clock if start.nil? || (previous_clock && clock - previous_clock > @interval)
|
|
148
|
+
maximum = [maximum, clock - start].max
|
|
149
|
+
else
|
|
150
|
+
start = nil
|
|
151
|
+
end
|
|
152
|
+
previous_clock = clock
|
|
153
|
+
end
|
|
154
|
+
maximum / 60.0
|
|
155
|
+
end
|
|
156
|
+
|
|
157
|
+
def low_days(days, capacities)
|
|
158
|
+
previous = nil
|
|
159
|
+
run = 0
|
|
160
|
+
maximum = 0
|
|
161
|
+
days.sort.each do |date, samples|
|
|
162
|
+
full_day = samples.size * @interval == @calendar.daily_seconds
|
|
163
|
+
complete = samples.all? { |inbound, outbound| !inbound.nil? && !outbound.nil? }
|
|
164
|
+
low = full_day && complete && samples.all? { |inbound, outbound| inbound / capacities[0] <= 0.2 && outbound / capacities[1] <= 0.2 }
|
|
165
|
+
run = if low
|
|
166
|
+
previous && @calendar.consecutive_workdays?(previous, date) ? run + 1 : 1
|
|
167
|
+
else
|
|
168
|
+
0
|
|
169
|
+
end
|
|
170
|
+
maximum = [maximum, run].max
|
|
171
|
+
previous = date
|
|
172
|
+
end
|
|
173
|
+
maximum
|
|
174
|
+
end
|
|
175
|
+
|
|
176
|
+
def maximum_utilization(summary)
|
|
177
|
+
values = %w[in_p95_utilization out_p95_utilization].filter_map { |key| summary[key] }
|
|
178
|
+
values.max || 0
|
|
179
|
+
end
|
|
180
|
+
|
|
181
|
+
def low_window?(summary, threshold)
|
|
182
|
+
summary.is_a?(Hash) && summary["status"] == "ok" &&
|
|
183
|
+
%w[in_p95_utilization out_p95_utilization].all? { |key| summary[key] && summary[key] <= threshold }
|
|
184
|
+
end
|
|
185
|
+
|
|
186
|
+
def decision(action, reason, notices = [])
|
|
187
|
+
Values.copy({ "action" => action, "reason" => reason, "notices" => notices })
|
|
188
|
+
end
|
|
189
|
+
end
|
|
190
|
+
end
|
|
191
|
+
end
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Netstack
|
|
4
|
+
module Circuits
|
|
5
|
+
class Endpoint < Data.define(:side, :host_id, :technical_host, :interface_name, :address, :layer3, :managed)
|
|
6
|
+
def initialize(side:, host_id: nil, technical_host: nil, interface_name: nil, address: nil, layer3: false, managed: true)
|
|
7
|
+
side = side.to_s
|
|
8
|
+
Monitoring::Inputs.invalid!("Circuit side must be a or z") unless %w[a z].include?(side)
|
|
9
|
+
Monitoring::Inputs.boolean(managed)
|
|
10
|
+
Monitoring::Inputs.boolean(layer3)
|
|
11
|
+
if managed
|
|
12
|
+
host_id = Monitoring::Inputs.identifier(host_id)
|
|
13
|
+
technical_host = Monitoring::Inputs.text(technical_host, name: "technical host")
|
|
14
|
+
Monitoring::Inputs.invalid!("Invalid technical host") if technical_host.match?(%r{[/\[\](),]})
|
|
15
|
+
interface_name = Monitoring::Inputs.text(interface_name, name: "interface name")
|
|
16
|
+
elsif host_id || technical_host || interface_name
|
|
17
|
+
Monitoring::Inputs.invalid!("External endpoints cannot identify a managed interface")
|
|
18
|
+
end
|
|
19
|
+
address = Monitoring::Inputs.ip(address) unless address.nil?
|
|
20
|
+
super(side: side.freeze, host_id: host_id&.freeze, technical_host: technical_host&.freeze,
|
|
21
|
+
interface_name: interface_name&.freeze, address: address&.freeze, layer3: layer3, managed: managed)
|
|
22
|
+
end
|
|
23
|
+
end
|
|
24
|
+
end
|
|
25
|
+
end
|
|
@@ -0,0 +1,212 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Netstack
|
|
4
|
+
module Circuits
|
|
5
|
+
# 读取提供方监控项和归属明确的触发器,构造 Monitoring::Plan。
|
|
6
|
+
# 规划不执行远端写入;原宿主 ID 限定迁移后的清理范围。
|
|
7
|
+
class MonitoringPlan
|
|
8
|
+
PATTERNS = {
|
|
9
|
+
"in" => /(?:net\.if\.in\[|ifhcin|bits? received|inbound|incoming|入方向速率|接收速率)/i,
|
|
10
|
+
"out" => /(?:net\.if\.out\[|ifhcout|bits? sent|outbound|outgoing|出方向速率|发送速率)/i,
|
|
11
|
+
"speed" => /(?:net\.if\.speed|ifhighspeed|interface speed|接口速率)/i,
|
|
12
|
+
"status" => /(?:net\.if\.(?:status|oper)|ifoperstatus|operational status|interface status|运行状态|接口状态)/i
|
|
13
|
+
}.freeze
|
|
14
|
+
DEFAULT_THRESHOLDS = { "high" => 0.95, "high_recovery" => 0.75, "low_bps" => 50, "low_recovery_bps" => 1_000 }.freeze
|
|
15
|
+
|
|
16
|
+
def initialize(client:)
|
|
17
|
+
@client = client
|
|
18
|
+
end
|
|
19
|
+
|
|
20
|
+
def call(key:, name:, bandwidth_mbps:, endpoints:, previous_host_ids: [], thresholds: {})
|
|
21
|
+
key = Monitoring::Inputs.text(key, name: "circuit key")
|
|
22
|
+
name = Monitoring::Inputs.text(name, name: "circuit name")
|
|
23
|
+
capacity = Monitoring::Inputs.number(bandwidth_mbps, positive: true) * 1_000_000
|
|
24
|
+
unless endpoints.is_a?(Array) && (1..2).cover?(endpoints.size) && endpoints.all?(Endpoint) && endpoints.map(&:side).uniq.size == endpoints.size
|
|
25
|
+
Monitoring::Inputs.invalid!("Circuit requires distinct a/z endpoints")
|
|
26
|
+
end
|
|
27
|
+
managed = endpoints.select(&:managed)
|
|
28
|
+
Monitoring::Inputs.invalid!("A managed circuit endpoint is required") if managed.empty?
|
|
29
|
+
identities = managed.map { |endpoint| [endpoint.host_id, endpoint.interface_name.downcase] }
|
|
30
|
+
Monitoring::Inputs.invalid!("Circuit endpoints must identify distinct interfaces") unless identities.uniq.size == identities.size
|
|
31
|
+
limits = thresholds!(thresholds)
|
|
32
|
+
host_ids = managed.map(&:host_id).uniq
|
|
33
|
+
old_host_ids = Monitoring::Inputs.ids(previous_host_ids)
|
|
34
|
+
touched_host_ids = old_host_ids | host_ids
|
|
35
|
+
owned = touched_host_ids.to_h { |host_id| [host_id, owned_triggers(host_id, key)] }
|
|
36
|
+
operations = cleanup(owned, managed)
|
|
37
|
+
managed.each do |endpoint|
|
|
38
|
+
items = @client.query("item", "hostids" => [endpoint.host_id], "monitored" => true)
|
|
39
|
+
selected = PATTERNS.to_h { |kind, pattern| [kind, select_item(items, endpoint.interface_name, pattern)] }
|
|
40
|
+
unless (selected["in"] || selected["out"]) && selected["speed"] && selected["status"]
|
|
41
|
+
Monitoring::Inputs.invalid!("Circuit traffic, speed and status items are required")
|
|
42
|
+
end
|
|
43
|
+
selected.each_value { |item| safe_key!(item.fetch("key_")) if item }
|
|
44
|
+
existing = owned.fetch(endpoint.host_id).select { |trigger| tag(trigger, "netstack.side") == endpoint.side }
|
|
45
|
+
by_kind = existing.to_h { |trigger| [tag(trigger, "netstack.kind"), trigger] }
|
|
46
|
+
Monitoring::Inputs.invalid!("Duplicate managed trigger kind") unless by_kind.size == existing.size && !by_kind.key?(nil)
|
|
47
|
+
previous_status_id = by_kind["interface_status"]&.fetch("triggerid")
|
|
48
|
+
peer = endpoints.find { |candidate| candidate.side != endpoint.side }
|
|
49
|
+
reachability_key = append_reachability(operations, endpoint, peer, name)
|
|
50
|
+
definitions(endpoint, selected, capacity, limits, reachability_key).each do |kind, expressions|
|
|
51
|
+
operation_key = "#{endpoint.side}:#{kind}"
|
|
52
|
+
dependencies = kind == "interface_status" ? [] : [{ "triggerid" => { "$ref" => "#{endpoint.side}:interface_status" } }]
|
|
53
|
+
attributes = {
|
|
54
|
+
"description" => "#{name} | #{endpoint.side.upcase} | #{kind}",
|
|
55
|
+
"expression" => expressions.first, "recovery_mode" => 1, "recovery_expression" => expressions.last,
|
|
56
|
+
"status" => 0, "priority" => 4, "dependencies" => dependencies,
|
|
57
|
+
"tags" => [{ "tag" => "netstack.circuit", "value" => key },
|
|
58
|
+
{ "tag" => "netstack.side", "value" => endpoint.side }, { "tag" => "netstack.kind", "value" => kind }]
|
|
59
|
+
}
|
|
60
|
+
current = by_kind.delete(kind)
|
|
61
|
+
if current
|
|
62
|
+
previous_dependencies = dependency_rows!(current)
|
|
63
|
+
attributes["dependencies"] = previous_dependencies.reject { |row| row["triggerid"].to_s == previous_status_id.to_s } + dependencies
|
|
64
|
+
# Only ownership tags are replaced; unrelated annotations survive.
|
|
65
|
+
attributes["tags"] = valid_tags!(current).reject do |row|
|
|
66
|
+
%w[netstack.circuit netstack.side netstack.kind].include?(row["tag"])
|
|
67
|
+
end + attributes["tags"]
|
|
68
|
+
before = current.slice("expression", "recovery_expression", "status", "tags", "dependencies")
|
|
69
|
+
operations << Monitoring::Mutation.new(key: operation_key, resource: "trigger", action: "update",
|
|
70
|
+
id: current.fetch("triggerid"), attributes: attributes, before: before)
|
|
71
|
+
else
|
|
72
|
+
operations << Monitoring::Mutation.new(key: operation_key, resource: "trigger", action: "create", attributes: attributes,
|
|
73
|
+
selector: trigger_selector(endpoint.host_id, key, endpoint.side, kind))
|
|
74
|
+
end
|
|
75
|
+
end
|
|
76
|
+
by_kind.each_value { |trigger| operations << disable_trigger(trigger, "#{endpoint.side}:obsolete:#{trigger.fetch("triggerid")}") }
|
|
77
|
+
end
|
|
78
|
+
Monitoring::Plan.new(key: "circuit:#{key}", operations: operations,
|
|
79
|
+
managed: { "circuit" => key, "host_ids" => host_ids, "previous_host_ids" => old_host_ids,
|
|
80
|
+
"touched_host_ids" => touched_host_ids })
|
|
81
|
+
end
|
|
82
|
+
|
|
83
|
+
private
|
|
84
|
+
|
|
85
|
+
def thresholds!(input)
|
|
86
|
+
input = Monitoring::Inputs.object(input)
|
|
87
|
+
Monitoring::Inputs.invalid!("Unknown circuit threshold") unless (input.keys - DEFAULT_THRESHOLDS.keys).empty?
|
|
88
|
+
limits = DEFAULT_THRESHOLDS.merge(input).transform_values { |value| Monitoring::Inputs.number(value, positive: true) }
|
|
89
|
+
unless limits["high"] <= 1 && limits["high_recovery"] < limits["high"] && limits["low_bps"] < limits["low_recovery_bps"]
|
|
90
|
+
Monitoring::Inputs.invalid!("Circuit thresholds require recovery hysteresis")
|
|
91
|
+
end
|
|
92
|
+
limits
|
|
93
|
+
end
|
|
94
|
+
|
|
95
|
+
def select_item(items, interface, pattern)
|
|
96
|
+
matches = items.select do |item|
|
|
97
|
+
next false unless item["key_"].is_a?(String) && item["name"].is_a?(String)
|
|
98
|
+
|
|
99
|
+
searchable = [item["key_"], item["name"], item["snmp_oid"]].compact.join(" ")
|
|
100
|
+
identity = [item["key_"], item["name"]].join(" ")
|
|
101
|
+
# An ifIndex or the numeric prefix of an OID does not identify a port.
|
|
102
|
+
searchable.match?(pattern) && identity.match?(%r{(?:\A|[^a-z0-9/_.-])#{Regexp.escape(interface)}(?=[^a-z0-9/_.-]|\z)}i)
|
|
103
|
+
end
|
|
104
|
+
Monitoring::Inputs.invalid!("Ambiguous circuit monitoring item") if matches.size > 1
|
|
105
|
+
matches.first
|
|
106
|
+
end
|
|
107
|
+
|
|
108
|
+
def definitions(endpoint, items, capacity, limits, reachability_key)
|
|
109
|
+
host = endpoint.technical_host
|
|
110
|
+
traffic = [items["in"], items["out"]].compact.map { |item| item.fetch("key_") }
|
|
111
|
+
speed = items.fetch("speed").fetch("key_")
|
|
112
|
+
status = items.fetch("status").fetch("key_")
|
|
113
|
+
values = {
|
|
114
|
+
"interface_status" => ["last(/#{host}/#{status})<>1", "last(/#{host}/#{status})=1"],
|
|
115
|
+
"bandwidth_high" => [traffic_expression(host, traffic, speed, "25m", ">", capacity * limits["high"], "or"),
|
|
116
|
+
traffic_expression(host, traffic, speed, "15m", "<", capacity * limits["high_recovery"], "and")],
|
|
117
|
+
"bandwidth_low" => [traffic_expression(host, traffic, speed, "5m", "<", limits["low_bps"], "or"),
|
|
118
|
+
traffic_expression(host, traffic, speed, "5m", ">", limits["low_recovery_bps"], "or")]
|
|
119
|
+
}
|
|
120
|
+
values["circuit_reachability"] = ["max(/#{host}/#{reachability_key},5m)=0", "min(/#{host}/#{reachability_key},2m)=1"] if reachability_key
|
|
121
|
+
values
|
|
122
|
+
end
|
|
123
|
+
|
|
124
|
+
def traffic_expression(host, keys, speed, interval, operator, threshold, joiner)
|
|
125
|
+
value = threshold.to_i == threshold ? threshold.to_i : threshold.round(4)
|
|
126
|
+
keys.map { |key| "((avg(/#{host}/#{key},#{interval})#{operator}#{value}) and last(/#{host}/#{speed})>0)" }.join(" #{joiner} ")
|
|
127
|
+
end
|
|
128
|
+
|
|
129
|
+
def append_reachability(operations, endpoint, peer, name)
|
|
130
|
+
return unless endpoint.layer3 && peer&.address
|
|
131
|
+
|
|
132
|
+
key = "icmpping[#{peer.address},4,200,,1000]"
|
|
133
|
+
selector = { "hostids" => [endpoint.host_id], "filter" => { "key_" => [key] } }
|
|
134
|
+
current = @client.find("item", selector)
|
|
135
|
+
if current
|
|
136
|
+
Monitoring::Inputs.invalid!("Reachability item is disabled or belongs to another key") unless current["key_"] == key && current["status"].to_s == "0"
|
|
137
|
+
else
|
|
138
|
+
operations << Monitoring::Mutation.new(key: "#{endpoint.side}:reachability_item", resource: "item", action: "create", selector: selector,
|
|
139
|
+
attributes: { "hostid" => endpoint.host_id, "name" => "#{name} peer reachability", "key_" => key,
|
|
140
|
+
"type" => 3, "value_type" => 3, "delay" => "1m", "history" => "30d", "trends" => "365d", "status" => 0 })
|
|
141
|
+
end
|
|
142
|
+
key
|
|
143
|
+
end
|
|
144
|
+
|
|
145
|
+
def cleanup(owned, endpoints)
|
|
146
|
+
retained = endpoints.each_with_object({}) do |endpoint, found|
|
|
147
|
+
owned.fetch(endpoint.host_id).each do |trigger|
|
|
148
|
+
found[trigger.fetch("triggerid").to_s] = true if tag(trigger, "netstack.side") == endpoint.side
|
|
149
|
+
end
|
|
150
|
+
end
|
|
151
|
+
seen = {}
|
|
152
|
+
owned.flat_map do |host_id, triggers|
|
|
153
|
+
triggers.filter_map do |trigger|
|
|
154
|
+
id = trigger.fetch("triggerid").to_s
|
|
155
|
+
next if retained[id] || seen[id]
|
|
156
|
+
|
|
157
|
+
seen[id] = true
|
|
158
|
+
disable_trigger(trigger, "cleanup:#{host_id}:#{trigger.fetch("triggerid")}")
|
|
159
|
+
end
|
|
160
|
+
end
|
|
161
|
+
end
|
|
162
|
+
|
|
163
|
+
def disable_trigger(trigger, key)
|
|
164
|
+
Monitoring::Mutation.new(key: key, resource: "trigger", action: "update", id: trigger.fetch("triggerid"),
|
|
165
|
+
attributes: { "status" => 1 }, before: trigger.slice("status", "tags"))
|
|
166
|
+
end
|
|
167
|
+
|
|
168
|
+
def owned_triggers(host_id, circuit_key)
|
|
169
|
+
@client.query("trigger", trigger_selector(host_id, circuit_key)).select do |trigger|
|
|
170
|
+
# Never trust an external filter alone to authorize cleanup.
|
|
171
|
+
tag(trigger, "netstack.circuit") == circuit_key
|
|
172
|
+
end
|
|
173
|
+
end
|
|
174
|
+
|
|
175
|
+
def trigger_selector(host_id, circuit_key, side = nil, kind = nil)
|
|
176
|
+
tags = [{ "tag" => "netstack.circuit", "value" => circuit_key, "operator" => 1 }]
|
|
177
|
+
tags << { "tag" => "netstack.side", "value" => side, "operator" => 1 } if side
|
|
178
|
+
tags << { "tag" => "netstack.kind", "value" => kind, "operator" => 1 } if kind
|
|
179
|
+
{ "hostids" => [host_id], "tags" => tags, "evaltype" => 0 }
|
|
180
|
+
end
|
|
181
|
+
|
|
182
|
+
def valid_tags!(trigger)
|
|
183
|
+
rows = trigger["tags"]
|
|
184
|
+
unless rows.is_a?(Array) && rows.all? { |row| row.is_a?(Hash) && row["tag"].is_a?(String) && row["value"].is_a?(String) }
|
|
185
|
+
Monitoring::Inputs.invalid!("Invalid trigger ownership tags")
|
|
186
|
+
end
|
|
187
|
+
rows
|
|
188
|
+
end
|
|
189
|
+
|
|
190
|
+
def dependency_rows!(trigger)
|
|
191
|
+
rows = trigger["dependencies"]
|
|
192
|
+
unless rows.is_a?(Array) && rows.all? { |row| row.is_a?(Hash) && row.key?("triggerid") }
|
|
193
|
+
Monitoring::Inputs.invalid!("Incomplete trigger dependencies")
|
|
194
|
+
end
|
|
195
|
+
ids = Monitoring::Inputs.ids(rows.map { |row| row["triggerid"] })
|
|
196
|
+
ids.map { |id| { "triggerid" => id } }
|
|
197
|
+
end
|
|
198
|
+
|
|
199
|
+
def tag(trigger, name)
|
|
200
|
+
values = valid_tags!(trigger).select { |row| row["tag"] == name }.map { |row| row["value"] }.uniq
|
|
201
|
+
Monitoring::Inputs.invalid!("Ambiguous trigger ownership") if values.size > 1
|
|
202
|
+
values.first
|
|
203
|
+
end
|
|
204
|
+
|
|
205
|
+
def safe_key!(value)
|
|
206
|
+
return if value.is_a?(String) && value.match?(%r{\A[a-zA-Z0-9_.]+(?:\[[a-zA-Z0-9_.,:"\-/ ]*\])?\z})
|
|
207
|
+
|
|
208
|
+
Monitoring::Inputs.invalid!("Unsafe item key in trigger expression")
|
|
209
|
+
end
|
|
210
|
+
end
|
|
211
|
+
end
|
|
212
|
+
end
|