telemetry-snmp 0.1.0

data/NOTICE.txt

@@ -0,0 +1,9 @@
+Telemetry::Snmp
+Copyright 2021 Optum
+
+Project Description:
+====================
+A Telemetry gem used to collect SNMP metrics and output them to Telemetry::AMQP in line protocol format
+
+Author(s):
+Esity

data/README.md

@@ -0,0 +1,54 @@
+# Telemetry::Snmp
+
+### Telemetry::Snmp Collector Service
+This is a servce that runs and collects SNMP metrics at whatever frequency is defined inside the basement. As you need
+additional collectors, you can scale this out to give you more parallel workers
+
+You can start this service by running
+```ruby
+bundle update
+bundle exec exe/snmp_collector
+```
+
+
+### Telemetry::Snmp::API
+The API allows for you to remotely CRUD devices, oids, users, device credentials, etc  
+The Routes are available via a [postman json](https://github.com/Optum/telemetry-snmp/blob/main/telemetry-snmp.json)  
+
+## OID Mappings
+The oid_walks table is the most utilized and probably what you are looking for. 
+Table Layout
+```json
+{
+  "oid": "the oid you want to grab metrics from(walk)",
+  "oid_index": "the oid to use as an index for naming",
+  "measurement_name": "what name to name the measurement",
+  "active": 1
+}
+```
+
+
+## Settings
+Telemetry::Snmp::Publisher
+```ruby
+ENV['telemetry.snmp.amqp.username'] = 'guest'
+ENV['telemetry.snmp.amqp.password'] = 'guest'
+ENV['telemetry.snmp.amqp.nodes'] = 'localhost'
+ENV['telemetry.snmp.amqp.vhost'] = 'telemetry'
+ENV['telemetry.snmp.amqp.port'] = '5672'
+ENV['telemetry.snmp.amqp.use_ssl'] = 'false'
+ENV['telemetry.snmp.amqp.exchange_name'] = 'telemetry.snmp'
+```
+
+Telemetry::Snmp::Data
+```ruby
+ENV['telemetry.snmp.data.adapter'] = 'mysql2'
+ENV['telemetry.snmp.data.username'] = 'root'
+ENV['telemetry.snmp.data.password'] = ''
+ENV['telemetry.snmp.data.database'] = 'telemetry_snmp'
+ENV['telemetry.snmp.data.host'] = '127.0.0.1'
+ENV['telemetry.snmp.data.port'] = '3306'
+ENV['telemetry.snmp.data.max_connections'] = '16'
+ENV['telemetry.snmp.data.pool_timeout'] = '2'
+ENV['telemetry.snmp.data.preconnect'] = 'concurrently'
+```

data/attribution.txt

@@ -0,0 +1 @@
+Add attributions here.

data/config.ru

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+require 'socket'
+
+require 'puma'
+require 'rack'
+require 'sinatra'
+require 'oj'
+require 'multi_json'
+
+require 'telemetry/snmp'
+require 'telemetry/snmp/api'
+
+Telemetry::Snmp::Data.start!
+run Telemetry::Snmp::API

data/exe/snmp_collector

@@ -0,0 +1,55 @@
+#!/usr/bin/env ruby
+
+require 'telemetry/snmp'
+Telemetry::Snmp.bootstrap
+
+trap('SIGTERM') { @quit = true }
+trap('SIGHUP') { @quit = true }
+trap('SIGINT') { @quit = true }
+
+@quit = false
+until @quit
+  @lines = []
+  Telemetry::Snmp::Data::Model::Device.each do |row|
+    next if row.values[:last_polled].to_i + row.values[:frequency] > Time.now.to_i
+
+    fields = {}
+    tags = {
+      hostname: row.values[:hostname],
+      ip_address: row.values[:ip_address],
+      env: row.values[:environment],
+      dc: row.values[:datacenter],
+      zone: row.values[:zone],
+      influxdb_node_group: 'snmp'
+    }
+
+    Telemetry::Snmp::Data::Model::OID.each do |oid_row|
+      oid_value = Telemetry::Snmp::Client.oid_value(row[:hostname], oid_row.values[:oid])
+      next if oid_value.nil?
+      next unless oid_value.is_a?(Integer) || oid_value.is_a?(Float)
+
+      fields[oid_row.values[:name]] = "#{Telemetry::Snmp::Client.oid_value(row[:hostname], oid_row.values[:oid])}i"
+    rescue StandardError => e
+      Telemetry::Logger.error "#{e.class}: #{e.message}"
+    end
+
+    @lines.push Telemetry::Metrics::Parser.to_line_protocol(
+      measurement: 'palo_alto_a',
+      fields: fields,
+      tags: tags,
+      timestamp: (DateTime.now.strftime('%Q').to_i * 1000 * 1000)
+    )
+
+    walker = Telemetry::Snmp::Client.grab_oid_metrics(row.values[:hostname])
+    Telemetry::Logger.info "Pushing #{walker.count} lines for #{row.values[:hostname]}" unless walker.empty?
+    Telemetry::Snmp::Publisher.push_lines(walker) unless walker.empty?
+
+    row.update(last_polled: Sequel::CURRENT_TIMESTAMP)
+    row.save
+  rescue StandardError => e
+    Telemetry::Logger.error "#{e.class}: #{e.message}"
+  end
+
+  Telemetry::Snmp::Publisher.push_lines(@lines) unless @lines.empty?
+  sleep(1)
+end

data/lib/telemetry/snmp.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+require 'telemetry/snmp/version'
+require 'telemetry/logger'
+require 'telemetry/metrics/parser'
+require 'telemetry/snmp/data'
+require 'telemetry/snmp/client'
+require 'telemetry/snmp/publisher'
+
+module Telemetry
+  module Snmp
+    class << self
+      def bootstrap
+        Telemetry::Logger.setup(level: 'info')
+        Telemetry::Logger.info "Starting Telemetry::Snmp v#{Telemetry::Snmp::VERSION}"
+        Telemetry::Snmp::Data.start!
+        Telemetry::Snmp::Client.load_mibs
+        Telemetry::Snmp::Publisher.start!
+        Telemetry::Logger.info 'Telemetry::Snmp bootstrapped!'
+      end
+    end
+  end
+end

data/lib/telemetry/snmp/api.rb

@@ -0,0 +1,55 @@
+require 'sinatra/base'
+require 'sinatra/contrib'
+require 'sinatra/multi_route'
+require 'sinatra/respond_with'
+require 'sinatra/custom_logger'
+require 'sinatra/namespace'
+require 'oj'
+
+require 'telemetry/snmp/controllers/device_creds'
+require 'telemetry/snmp/controllers/devices'
+require 'telemetry/snmp/controllers/oid_groups'
+require 'telemetry/snmp/controllers/oids'
+require 'telemetry/snmp/controllers/users'
+require 'telemetry/snmp/controllers/walks'
+
+module Telemetry
+  module Snmp
+    class API < Sinatra::Base
+      register Sinatra::JSON
+      register Sinatra::Namespace
+      register Sinatra::RespondWith
+
+      error do
+        content_type :json
+        status 500
+
+        { result: 'error', message: env['sinatra.error'].message }.to_json
+      end
+
+      before do
+        headers 'Access-Control-Allow-Origin' => '*',
+                'Access-Control-Allow-Methods' => %w[OPTIONS GET POST]
+      end
+
+      after do
+        content_type :json
+        response.body = Oj.dump(response.body, mode: :compat) unless response.body.is_a? String
+      end
+
+      get '/version' do
+        {
+          version: Telemetry::Snmp::VERSION,
+          migration_version: Telemetry::Snmp::Data.migration_version
+        }
+      end
+
+      namespace('/users') { register Telemetry::Snmp::Controller::Users }
+      namespace('/devices/creds') { register Telemetry::Snmp::Controller::DeviceCreds }
+      namespace('/devices') { register Telemetry::Snmp::Controller::Devices }
+      namespace('/oid_groups') { register Telemetry::Snmp::Controller::OIDGroups }
+      namespace('/oid') { register Telemetry::Snmp::Controller::OIDs }
+      namespace('/walks') { register Telemetry::Snmp::Controller::Walks }
+    end
+  end
+end

data/lib/telemetry/snmp/auth.rb

@@ -0,0 +1,54 @@
+require 'net/ldap'
+require 'telemetry/snmp/auth/defaults'
+
+module Telemetry
+  module Snmp
+    class Auth
+      include Telemetry::Snmp::AuthDefaults
+
+      def initialize(username:, **opts)
+        @username = username
+        @details = {}
+        @opts = opts
+      end
+
+      def process_result(result)
+        unless result.is_a? Net::LDAP::Entry
+          @success = false
+          return
+        end
+        @details[:username] = result.sAMAccountName.first
+        @details[:email] = result.mail.first
+        @details[:first] = result.givenName.first
+        @details[:last] = result.sn.first
+        @success = true
+      end
+
+      def search_user(ldap, username)
+        user_filter = Net::LDAP::Filter.eq('sAMAccountName', username)
+
+        ldap.search(base: treebase, filter: user_filter, attrs: attrs, return_result: false) do |entry|
+          @details[:group_access] = entry.memberof.include?("CN=#{admin_group}, #{treebase}")
+          return entry
+        end
+      end
+
+      def auth_with_service(password)
+        options = defaults
+        options[:auth] = defaults_auth
+        result = provider.new(options).bind_as(base: defaults[:base], attributes: attrs, filter: filter, password: password) # rubocop:disable Layout/LineLength
+        process_result(result.first)
+      end
+
+      def auth_without_service(password)
+        options = { host: defaults[:host], port: defaults[:port] }
+        options[:auth] = { password: password, username: @username, method: :simple }
+        ldap = provider.new(options)
+        @success = ldap.bind
+        return unless @success
+
+        process_result(search_user(ldap, @username))
+      end
+    end
+  end
+end

data/lib/telemetry/snmp/auth/defaults.rb

@@ -0,0 +1,41 @@
+module Telemetry
+  module Snmp
+    module AuthDefaults
+      def opts
+        @opts ||= {}
+      end
+
+      def attrs
+        %w[mail cn sn objectclass givenName sAMAccountName MemberOf]
+      end
+
+      def treebase
+        opts[:treebase] || ENV['treebase'] || 'CN=Users,DC=com'
+      end
+
+      def ldap_host
+        opts[:ldap_host] || ENV['ldap_host'] || 'localhost'
+      end
+
+      def ldap_port
+        opts[:ldap_port] || ENV['ldap_host'] || '389'
+      end
+
+      def provider
+        Net::LDAP
+      end
+
+      def filter(username = @username)
+        "(sAMAccountName=#{username})"
+      end
+
+      def admin_group
+        opts[:admin_group] || ENV['ldap_admin_group']
+      end
+
+      def users_group
+        opts[:users_group] || ENV['ldap_users_group']
+      end
+    end
+  end
+end

data/lib/telemetry/snmp/client.rb

@@ -0,0 +1,104 @@
+require 'socket'
+require 'netsnmp'
+
+module Telemetry
+  module Snmp
+    module Client
+      @connections = {}
+
+      class << self
+        def load_mibs
+          ENV['MIBDIRS'] = "#{__dir__}/mibs"
+          NETSNMP::MIB.load_defaults
+          # NETSNMP::MIB.load("#{__dir__}/mibs/PAN-COMMON-MIB.my")
+          # NETSNMP::MIB.load("#{__dir__}/mibs/PAN-ENTITY-EXT-MIB.my")
+          # NETSNMP::MIB.load("#{__dir__}/mibs/PAN-GLOBAL-REG-MIB.my")
+          # NETSNMP::MIB.load("#{__dir__}/mibs/PAN-GLOBAL-TC-MIB.my")
+          # NETSNMP::MIB.load("#{__dir__}/mibs/PAN-LC-MIB.my")
+          # NETSNMP::MIB.load("#{__dir__}/mibs/PAN-PRODUCT-MIB.my")
+        end
+
+        def connection(host)
+          return @connections[host.to_sym] if @connections.key? host.to_sym
+
+          dataset = Telemetry::Snmp::Data::Model::Device[hostname: host]
+
+          @connections[host.to_sym] = NETSNMP::Client.new(
+            host: dataset.values[:ip_address],
+            port: dataset.values[:port],
+            username: dataset.device_cred.values[:username],
+            auth_password: dataset.device_cred.values[:auth_password],
+            auth_protocol: dataset.device_cred.values[:auth_protocol].to_sym,
+            priv_password: dataset.device_cred.values[:priv_password],
+            priv_protocol: dataset.device_cred.values[:priv_protocol].to_sym,
+            security_level: dataset.device_cred.values[:security_level].to_sym
+          )
+        end
+
+        def oid_value(host, oid)
+          connection(host).get(oid: oid)
+        rescue StandardError
+          nil
+        end
+
+        def oid_walk(host, oid)
+          results = []
+          connection(host).walk(oid: oid).each do |oid_code, value|
+            hash = { oid_code: oid_code, value: value }
+            begin
+              ident = NETSNMP::MIB.identifier(oid_code)
+
+              hash[:identifier] = ident.first
+            rescue StandardError
+              # literally do nothing
+            end
+            results.push hash
+          end
+
+          results
+        end
+
+        def grab_oid_metrics(hostname)
+          device = Telemetry::Snmp::Data::Model::Device[hostname: hostname]
+          @lines = []
+          Telemetry::Snmp::Data::Model::OIDWalks.where(:active).each do |row|
+            index = {}
+            Telemetry::Snmp::Client.oid_walk(device.values[:hostname], row.values[:oid_index]).each do |hash|
+              index[hash[:oid_code].delete_prefix("#{row.values[:oid_index]}.")] = hash[:value].gsub(%r{\\/}, '.')
+            end
+
+            timestamp = DateTime.now.strftime('%Q').to_i * 1000 * 1000
+            Telemetry::Snmp::Client.oid_walk(device.values[:hostname], row.values[:oid_walk]).each do |walk|
+              key = walk[:oid_code].split('.').last
+              next if walk[:value].is_a? String
+              next if walk[:value].nil?
+
+              fields = {}
+              fields[walk[:identifier]] = "#{walk[:value]}i"
+              tags = {
+                hostname: device.values[:hostname],
+                interface: index[key],
+                ip_address: device.values[:ip_address],
+                zone: device.values[:zone],
+                env: device.values[:environment],
+                dc: device.values[:datacenter],
+                influxdb_node_group: 'snmp'
+              }
+
+              line = Telemetry::Metrics::Parser.to_line_protocol(
+                measurement: row.values[:measurement],
+                fields: fields,
+                tags: tags,
+                timestamp: timestamp
+              )
+
+              @lines.push line
+            end
+          end
+
+          @lines
+        end
+      end
+    end
+  end
+end

data/lib/telemetry/snmp/controllers/device_creds.rb

@@ -0,0 +1,105 @@
+require 'sinatra/extension'
+
+module Telemetry
+  module Snmp
+    module Controller
+      module DeviceCreds
+        extend Sinatra::Extension
+        # id, port, username, auth_password, auth_protocol, priv_password, priv_protocol, security_level
+
+        get '' do
+          results = {}
+
+          Telemetry::Snmp::Data::Model::DeviceCred.all.each do |creds|
+            results[creds.values[:id]] = {
+              id: creds.values[:id],
+              port: creds.values[:port],
+              username: creds.values[:username],
+              auth_protocol: creds.values[:auth_protcol],
+              priv_protocol: creds.values[:priv_protocol],
+              security_level: creds.values[:security_level],
+              created: creds.values[:created],
+              updated: creds.values[:updated]
+            }
+          end
+
+          results
+        end
+
+        get '/:id' do
+          cred = Telemetry::Snmp::Data::Model::DeviceCred[params[:id]]
+          if cred.nil?
+            status 404
+            return {}
+          end
+
+          results = cred.values.dup
+          results.delete(:auth_password)
+          results.delete(:priv_password)
+
+          results
+        end
+
+        post '' do
+          body = MultiJson.load(request.body.read, symbolize_keys: true)
+
+          if params[:username].nil? || params[:auth_password].nil?
+            status 400
+            {
+              error: true,
+              username_missing: params[:username].nil?,
+              auth_password_missing: params[:auth_password].nil?
+            }
+          end
+          insert = {
+            port: body[:port] || 161,
+            username: body[:username],
+            auth_password: body[:auth_password],
+            auth_protocol: body[:auth_protocol] || 'sha',
+            priv_password: body[:priv_password],
+            priv_protocol: body[:priv_protocol] || 'aes',
+            security_level: body[:security_level] || 'auth_priv',
+            created: Sequel::CURRENT_TIMESTAMP
+          }
+
+          { id: Telemetry::Snmp::Data::Model::DeviceCred.insert(**insert), username: insert[:username] }
+        end
+
+        put '/:id' do
+          status 405
+          { error: true, message: 'puts not supported', id: params[:id] }
+        end
+
+        patch '/:id' do
+          cred = Telemetry::Snmp::Data::Model::DeviceCred[params[:id]]
+          if cred.nil?
+            status 404
+            return { error: true, id: params[:id], message: "cannot find #{params[:id]}" }
+          end
+
+          body = MultiJson.load(request.body.read, symbolize_keys: true)
+          update = {}
+          fields = %i[port username auth_password auth_protocol priv_password priv_protocol security_level]
+          fields.each { |field| update[field] = body[field] if body.key? field }
+
+          if update.empty?
+            status 400
+            return { error: true, message: 'no valid fields to update' }
+          end
+
+          update[:updated] = Sequel::CURRENT_TIMESTAMP
+          cred.update(**update)
+          { error: false, updated_field: update.keys, id: params[:id] }
+        end
+
+        delete '/:id' do
+          cred = Telemetry::Snmp::Data::Model::DeviceCred[params[:id]]
+          status 404 if cred.nil?
+          return { error: true, message: "#{params[:id]} not found" } if cred.nil?
+
+          { error: !result.delete, id: params[:id] }
+        end
+      end
+    end
+  end
+end