telemetry-snmp 0.2.0 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 1b1cf78bbce21aa906630c1e4430ed67805d3fa1332664c0048976ea12c92746
-  data.tar.gz: 0365001cb5dd2dd1767c76234e485b18e0f4c5727743489459b69e1832a478bc
+  metadata.gz: ed97642826f1f7eae99d08d99d6e98a587a1f2427b7fb35c1fe3bf58a1056118
+  data.tar.gz: 5cbf336cb9c75b0e89b578f85a89c2874a6ce4cbaa6662d48edfeb34058ee4a0
 SHA512:
-  metadata.gz: 7dbd63ecd438e885bc22080aed5cf5044a7344671600134d93d77d07c89d6a2065b3d1558cfd1b30e84b44e8a718d581ad08d6436ed4af852c16ec43f14715d3
-  data.tar.gz: f3377fc61d505f6b165f8d85709d2ea7471933f4d3c83ecb2f50d4360c2e992dbc547114d722f344b65a72b1f94db1f633ba445df3bd82780bd9563b6d11b0cd
+  metadata.gz: e1692d91324dfa8eb117c81c78d908dc8018e4ccd7e5682feb6bc45a0dd43c884974db9900fddf54a9e9ae5100d11e41f49c20a0ff70b76bb435fd1447708348
+  data.tar.gz: 3e9d11ad581fb35eba0e068d61f4d18dfbc8ddcce599392a903ac645f35647fa300148ebaaaa7ac0ff23a22c7efba176630c11db654329587648df132f1e29d1

data/CHANGELOG.md

@@ -1,5 +1,16 @@
 # Telemetry::SNMP
 
+## [0.3.0]
+* Updating locking mechanism
+* Allow for parallel collections
+* Adding measurement prefixes
+
+## [0.2.2]
+* Updating logic to use the IP address of the device first and then hostname if ip is null
+
+## [0.2.1]
+Updating logic so it updates the last_polled column at the beginning of the loop
+
 ## [0.2.0]
 * Fixing ENV variables to not use .
 * Adding new device lock table for future reference

data/README.md

@@ -11,6 +11,16 @@ bundle exec exe/snmp_collector
 ```
 
 
+1.3.6.1.2.1.47
+1.3.6.1.2.1.2.2.1.8 = Interface Up/Down
+1.3.6.1.2.1.2.2.1.10.1 = Interface in counter
+1.3.6.1.2.1.2.2.1.14 = Interface In Errors
+1.3.6.1.2.1.2.2.1.16 = Interface out counters
+1.3.6.1.2.1.2.2.1.20 = Inteface Out Errors
+1.3.6.1.2.1.31.1.1.1
+
+https://knowledgebase.paloaltonetworks.com/KCSArticleDetail?id=kA10g000000ClaSCAS
+
 ### Telemetry::Snmp::API
 The API allows for you to remotely CRUD devices, oids, users, device credentials, etc  
 The Routes are available via a [postman json](https://github.com/Optum/telemetry-snmp/blob/main/telemetry-snmp.json)  

data/lib/telemetry/snmp/api.rb

@@ -62,6 +62,10 @@ module Telemetry
         Telemetry::Snmp::Collector.loop_devices
       end
 
+      get '/poll_next_device' do
+        Telemetry::Snmp::Collector.poll_next_device
+      end
+
       namespace('/users') { register Telemetry::Snmp::Controller::Users }
       namespace('/devices/creds') { register Telemetry::Snmp::Controller::DeviceCreds }
       namespace('/devices') { register Telemetry::Snmp::Controller::Devices }

data/lib/telemetry/snmp/client.rb

@@ -21,10 +21,10 @@ module Telemetry
         def connection(host)
           return @connections[host.to_sym] if @connections.key? host.to_sym
 
-          dataset = Telemetry::Snmp::Data::Model::Device[hostname: host]
+          dataset = Telemetry::Snmp::Data::Model::Device.where(hostname: host).or(ip_address: host).first
 
           @connections[host.to_sym] = NETSNMP::Client.new(
-            host: dataset.values[:ip_address],
+            host: dataset.values[:hostname],
             port: dataset.values[:port],
             username: dataset.device_cred.values[:username],
             auth_password: dataset.device_cred.values[:auth_password],
@@ -59,7 +59,7 @@ module Telemetry
         end
 
         def grab_oid_metrics(hostname)
-          device = Telemetry::Snmp::Data::Model::Device[hostname: hostname]
+          device = Telemetry::Snmp::Data::Model::Device.where(hostname: hostname).or(ip_address: hostname).first
           @lines = []
           Telemetry::Snmp::Data::Model::OIDWalks.where(:active).each do |row|
             index = {}
@@ -94,6 +94,8 @@ module Telemetry
 
               @lines.push line
             end
+          rescue StandardError => e
+            Telemetry::Logger.error "#{e.class}: #{hostname}, #{e.message}"
           end
 
           @lines

data/lib/telemetry/snmp/collector.rb

@@ -2,16 +2,30 @@ module Telemetry
   module Snmp
     module Collector
       class << self
-        def worker_name
-          "#{::Socket.gethostname.tr('.', '_')}.#{::Process.pid}.#{Thread.current.object_id}"
+        def loop_devices
+          count = 0
+          Telemetry::Snmp::Data::Model::Device.where(:active).order(:last_polled).each do |row|
+            break if count >= 10
+            next if row.values[:last_polled].to_i + row.values[:frequency] > Time.now.to_i
+            next if device_locked?(row.values[:id])
+
+            Telemetry::Logger.info "Grabbing metrics for #{row.values[:hostname]}"
+            device = Telemetry::Snmp::DeviceCollector.new(row.values[:hostname])
+            device.async.collect
+            count += 1
+          end
         end
 
-        def loop_devices
-          Telemetry::Snmp::Data::Model::Device.where(:active).each do |row|
+        def poll_next_device
+          Telemetry::Snmp::Data::Model::Device.where(:active).order(:last_polled).each do |row|
             next if row.values[:last_polled].to_i + row.values[:frequency] > Time.now.to_i
             next if device_locked?(row.values[:id])
 
+            Telemetry::Logger.info "Grabbing metrics for #{row.values[:hostname]}"
             collect(row.values[:id])
+            break
+          rescue StandardError => e
+            Telemetry::Logger.exception(e, level: 'error')
           end
         end
 
@@ -19,81 +33,17 @@ module Telemetry
           Telemetry::Snmp::Data::Model::DeviceLock.each do |row|
             next if row.values[:expires] < Sequel::CURRENT_TIMESTAMP
 
+            Telemetry::Logger.warn "removing lock for #{row.values[:hostname]}"
             row.delete
           end
         end
 
         def device_locked?(device_id)
-          !Telemetry::Snmp::Data::Model::DeviceLock[device_id: device_id].nil?
-        end
-
-        def lock_device(device_id)
-          return false unless Telemetry::Snmp::Data::Model::DeviceLock[device_id: device_id].nil?
-
-          Telemetry::Snmp::Data::Model::DeviceLock.insert(
-            worker_name: worker_name,
-            device_id: device_id,
-            created: Sequel::CURRENT_TIMESTAMP,
-            expires: Sequel::CURRENT_TIMESTAMP
-          )
-          true
-        end
-
-        # noinspection RubyArgCount
-        def unlock_device(device_id)
-          device = Telemetry::Snmp::Data::Model::DeviceLock[device_id: device_id]
-          return true if device.nil?
-
-          device.delete
+          Telemetry::Snmp::Data::Model::DeviceLock.where(device_id: device_id).count.positive?
         end
 
-        def collect(device_id)
-          lock_device(device_id)
-          row = Telemetry::Snmp::Data::Model::Device[device_id]
-          lines = []
-          fields = {}
-          tags = {
-            hostname: row.values[:hostname],
-            ip_address: row.values[:ip_address],
-            env: row.values[:environment],
-            dc: row.values[:datacenter],
-            zone: row.values[:zone],
-            influxdb_node_group: 'snmp',
-            influxdb_database: 'snmp'
-          }
-
-          Telemetry::Snmp::Data::Model::OID.each do |oid_row|
-            break if @quit
-
-            oid_value = Telemetry::Snmp::Client.oid_value(row[:hostname], oid_row.values[:oid])
-            next if oid_value.nil?
-            next unless oid_value.is_a?(Integer) || oid_value.is_a?(Float)
-
-            fields[oid_row.values[:name]] =
-              "#{Telemetry::Snmp::Client.oid_value(row[:hostname], oid_row.values[:oid])}i"
-          rescue StandardError => e
-            Telemetry::Logger.error "#{e.class}: #{e.message}"
-          end
-
-          lines.push Telemetry::Metrics::Parser.to_line_protocol(
-            measurement: 'palo_alto',
-            fields: fields,
-            tags: tags,
-            timestamp: (DateTime.now.strftime('%Q').to_i * 1000 * 1000)
-          )
-
-          walker = Telemetry::Snmp::Client.grab_oid_metrics(row.values[:hostname])
-          Telemetry::Logger.info "Pushing #{walker.count} lines for #{row.values[:hostname]}" unless walker.empty?
-          Telemetry::Snmp::Publisher.push_lines(walker) unless walker.empty?
-
-          row.update(last_polled: Sequel::CURRENT_TIMESTAMP)
-          row.save
-
-          Telemetry::Snmp::Publisher.push_lines(lines) unless lines.empty?
-          unlock_device(device_id)
-        rescue StandardError => e
-          Telemetry::Logger.exception(e, level: 'error')
-          unlock_device(device_id)
+        def device_unlocked?(device_id)
+          !device_locked?(device_id)
         end
       end
     end

data/lib/telemetry/snmp/data/migrations/012_add_measurement_prefix.rb

@@ -0,0 +1,8 @@
+Sequel.migration do
+  change do
+    alter_table(:oids) do
+      add_column :active, Integer, limit: 1, null: false, default: 1, index: true
+      add_column :measurement_name, String, null: false, default: 'snmp'
+    end
+  end
+end

data/lib/telemetry/snmp/data/migrations/013_add_fields_to_devices.rb

@@ -0,0 +1,9 @@
+Sequel.migration do
+  change do
+    alter_table(:oids) do
+      add_column :brand, String
+      add_column :type, String
+      add_column :role, String
+    end
+  end
+end

data/lib/telemetry/snmp/device_collector.rb

@@ -0,0 +1,186 @@
+module Telemetry
+  module Snmp
+    class DeviceCollector
+      include Concurrent::Async
+
+      def initialize(host)
+        @device = Telemetry::Snmp::Data::Model::Device.where(hostname: host).or(ip_address: host).first
+        return if @device.nil?
+
+        @id = @device.values[:id]
+        @hostname = @device.values[:hostname]
+      end
+
+      def connection
+        @connection ||= NETSNMP::Client.new(
+          host: @device.values[:hostname],
+          port: @device.values[:port],
+          username: @device.device_cred.values[:username],
+          auth_password: @device.device_cred.values[:auth_password],
+          auth_protocol: @device.device_cred.values[:auth_protocol].to_sym,
+          priv_password: @device.device_cred.values[:priv_password],
+          priv_protocol: @device.device_cred.values[:priv_protocol].to_sym,
+          security_level: @device.device_cred.values[:security_level].to_sym
+        )
+      end
+
+      def collect
+        return false unless lock_device
+
+        @collection_start = Time.now
+
+        @device.update(last_polled: Sequel::CURRENT_TIMESTAMP)
+        @device.save
+        @lines = []
+        @fields = {}
+
+        Telemetry::Snmp::Data::Model::OID.where(:active).each do |oid_row|
+          oid_value = oid_value(oid_row.values[:oid])
+          oid_value = oid_value.to_i if oid_value.is_a?(NETSNMP::Timetick)
+
+          if oid_value.nil?
+            Telemetry::Logger.warn "#{@hostname} nil result for #{oid_row.values[:oid]}"
+            next
+          end
+
+          unless oid_value.is_a?(Integer) || oid_value.is_a?(Float)
+            Telemetry::Logger.error "#{@hostname} nil result for #{oid_row.values[:oid]} class: #{oid_value.class}, #{oid_value}" # rubocop:disable Layout/LineLength
+            next
+          end
+
+          @fields[oid_row.values[:name]] = "#{oid_value}i"
+        rescue StandardError => e
+          Telemetry::Logger.error "#{e.class}: #{e.message}"
+        end
+
+        @lines.push Telemetry::Metrics::Parser.to_line_protocol(
+          measurement: 'palo_alto',
+          fields: @fields,
+          tags: tags,
+          timestamp: (DateTime.now.strftime('%Q').to_i * 1000 * 1000)
+        )
+
+        walker = grab_oid_metrics
+        unless walker.empty?
+          Telemetry::Logger.info "Pushing #{walker.count} lines for #{@hostname} in #{((Time.now - @collection_start) * 1000).round}ms" # rubocop:disable Layout/LineLength
+        end
+        Telemetry::Snmp::Publisher.push_lines(walker) unless walker.empty?
+        Telemetry::Snmp::Publisher.push_lines(@lines) unless @lines.empty?
+        unlock_device
+      rescue StandardError => e
+        Telemetry::Logger.exception(e, level: 'error')
+        unlock_device
+      end
+
+      def oid_value(oid)
+        connection.get(oid: oid)
+      rescue StandardError
+        nil
+      end
+
+      def oid_walk(oid)
+        results = []
+        connection.walk(oid: oid).each do |oid_code, value|
+          hash = { oid_code: oid_code, value: value }
+          begin
+            ident = NETSNMP::MIB.identifier(oid_code)
+
+            hash[:identifier] = ident.first
+          rescue StandardError
+            # literally do nothing
+          end
+          results.push hash
+        end
+
+        results
+      end
+
+      def grab_oid_metrics
+        @lines = []
+        Telemetry::Snmp::Data::Model::OIDWalks.where(:active).each do |row|
+          index = {}
+          oid_walk(row.values[:oid_index]).each do |hash|
+            index[hash[:oid_code].delete_prefix("#{row.values[:oid_index]}.")] = hash[:value].gsub(%r{\\/}, '.')
+          end
+
+          timestamp = DateTime.now.strftime('%Q').to_i * 1000 * 1000
+          oid_walk(row.values[:oid_walk]).each do |walk|
+            key = walk[:oid_code].split('.').last
+            next if walk[:value].is_a? String
+            next if walk[:value].nil?
+
+            fields = {}
+            fields[walk[:identifier]] = "#{walk[:value]}i"
+            tags = {
+              hostname: @hostname,
+              interface: index[key],
+              ip_address: @device.values[:ip_address],
+              zone: @device.values[:zone],
+              env: @device.values[:environment],
+              dc: @device.values[:datacenter],
+              influxdb_node_group: 'snmp',
+              influxdb_database: 'snmp'
+            }
+
+            line = Telemetry::Metrics::Parser.to_line_protocol(
+              measurement: "snmp_#{row.values[:measurement_name]}",
+              fields: fields,
+              tags: tags,
+              timestamp: timestamp
+            )
+
+            @lines.push line
+          end
+        rescue StandardError => e
+          Telemetry::Logger.error "#{e.class}: #{@hostname}, #{e.message}"
+        end
+
+        @lines
+      end
+
+      def device_locked?
+        Telemetry::Snmp::Data::Model::DeviceLock.where(device_id: @id).count.positive?
+      end
+
+      def device_unlocked?
+        Telemetry::Snmp::Data::Model::DeviceLock.where(device_id: @id).count.zero?
+      end
+
+      def lock_device
+        Telemetry::Snmp::Data::Model::DeviceLock.insert(
+          worker_name: worker_name,
+          device_id: @id,
+          created: Sequel::CURRENT_TIMESTAMP,
+          expires: Sequel::CURRENT_TIMESTAMP
+        )
+        true
+      end
+
+      def tags
+        tags = {
+          hostname: @device.values[:hostname],
+          ip_address: @device.values[:ip_address],
+          env: @device.values[:environment],
+          dc: @device.values[:datacenter],
+          zone: @device.values[:zone],
+          influxdb_node_group: 'snmp',
+          influxdb_database: 'snmp'
+        }
+        tags.delete_if { |_k, v| v.nil? }
+
+        tags
+      end
+
+      def unlock_device
+        device = Telemetry::Snmp::Data::Model::DeviceLock[device_id: @id]
+        return true if device.nil?
+
+        device.delete
+      end
+
+      def worker_name
+        "#{::Socket.gethostname.tr('.', '_')}.#{::Process.pid}.#{Thread.current.object_id}"
+      end
+    end
+  end
+end

data/lib/telemetry/snmp/publisher.rb

@@ -118,7 +118,7 @@ module Telemetry
             routing_key: 'snmp',
             persistent: true,
             mandatory: false,
-            timstamp: Time.now.to_i,
+            timestamp: Time.now.to_i,
             type: 'metric',
             content_type: 'application/json',
             content_encoding: 'identity'

data/lib/telemetry/snmp/version.rb

@@ -2,6 +2,6 @@
 
 module Telemetry
   module Snmp
-    VERSION = '0.2.0'
+    VERSION = '0.3.0'
   end
 end

data/lib/telemetry/snmp.rb

@@ -7,6 +7,7 @@ require 'telemetry/snmp/data'
 require 'telemetry/snmp/client'
 require 'telemetry/snmp/publisher'
 require 'telemetry/snmp/collector'
+require 'telemetry/snmp/device_collector'
 
 module Telemetry
   module Snmp
@@ -18,6 +19,30 @@ module Telemetry
         Telemetry::Snmp::Client.load_mibs
         Telemetry::Snmp::Publisher.start!
         Telemetry::Logger.info 'Telemetry::Snmp bootstrapped!'
+        start_expire_devices
+        start_collection
+      end
+
+      def start_expire_devices
+        @expire_devices_task = Concurrent::TimerTask.new(execution_interval: 300, timeout_interval: 10) do
+          Telemetry::Snmp::Collector.unlock_expired_devices
+        end
+        @expire_devices_task.execute
+      end
+
+      def stop_expire_devices
+        @expire_devices_task.stop
+      end
+
+      def start_collection
+        @collection_task = Concurrent::TimerTask.new(execution_interval: 10, timeout_interval: 300) do
+          Telemetry::Snmp::Collector.loop_devices
+        end
+        @collection_task.execute
+      end
+
+      def stop_collection
+        @collection_task.stop
       end
     end
   end

data/telemetry-snmp.gemspec

@@ -27,6 +27,7 @@ Gem::Specification.new do |spec|
   spec.add_dependency 'concurrent-ruby', '>= 1.1.7'
   spec.add_dependency 'concurrent-ruby-ext', '>= 1.1.7'
   spec.add_dependency 'connection_pool', '>= 2.2.3'
+  spec.add_dependency 'elastic-apm'
   spec.add_dependency 'faraday'
   spec.add_dependency 'faraday_middleware'
   spec.add_dependency 'faraday-request-timer'

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: telemetry-snmp
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.3.0
 platform: ruby
 authors:
 - Esity
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-09-23 00:00:00.000000000 Z
+date: 2021-10-30 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: concurrent-ruby
@@ -52,6 +52,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: 2.2.3
+- !ruby/object:Gem::Dependency
+  name: elastic-apm
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: faraday
   requirement: !ruby/object:Gem::Requirement
@@ -339,6 +353,8 @@ files:
 - lib/telemetry/snmp/data/migrations/009_create_tag_name_column.rb
 - lib/telemetry/snmp/data/migrations/010_create_user_audit_table.rb
 - lib/telemetry/snmp/data/migrations/011_device_locks.rb
+- lib/telemetry/snmp/data/migrations/012_add_measurement_prefix.rb
+- lib/telemetry/snmp/data/migrations/013_add_fields_to_devices.rb
 - lib/telemetry/snmp/data/models/device.rb
 - lib/telemetry/snmp/data/models/device_cred.rb
 - lib/telemetry/snmp/data/models/device_lock.rb
@@ -348,6 +364,7 @@ files:
 - lib/telemetry/snmp/data/models/oid_walk.rb
 - lib/telemetry/snmp/data/models/user.rb
 - lib/telemetry/snmp/data/models/user_audit_log.rb
+- lib/telemetry/snmp/device_collector.rb
 - lib/telemetry/snmp/mibs/AGENTX-MIB.txt
 - lib/telemetry/snmp/mibs/AIRPORT-BASESTATION-3-MIB.txt
 - lib/telemetry/snmp/mibs/BRIDGE-MIB.txt