netcrawl 0.0.1

data/.gitignore

@@ -0,0 +1,2 @@
+Gemfile.lock
+gems

data/Gemfile

@@ -0,0 +1,3 @@
+source 'https://rubygems.org'
+
+gemspec

data/README.md

@@ -0,0 +1,86 @@
+# Netcrawl
+Given snmp community and single device, crawls the network via discovering
+LLDP/CDP neighbours, while producing list or dot file (for graphviz digraphs)
+
+## Install
+% gem install netcrawl
+
+## Use
+% netcrawl --graphiz router.example.com
+
+## Command line
+```
+Usage: netcrawl [options] hostname
+    -g, --graphviz       dot output use 'dot -Tpng -o map.png map.dot'
+    -l, --list           list nodes
+    -j, --json           json output
+    -y, --yaml           yaml output
+    -a, --hash           hash/associative array output
+    -r, --resolve        resolve addresses to names
+    -p, --purge          remove peers not in configured CIDR
+    -c, --community      SNMP community to use
+    -d, --debug          turn debugging on
+    -h, --help           Display this help message.
+
+```
+  * graphiz   - graphis (dot) output
+  * list      - list nodes found
+  * json      - json output
+  * yaml      - yaml output
+  * hash      - ruby hash output
+  * resolve   - resolve IP addresses
+  * purge     - remove non-cidr matching peers from output
+  * community - sets snmp community
+  * debug     - turn on debugging
+
+## Config
+```
+---
+use:
+- LLDP
+- CDP
+poll:
+- 192.0.2.0/24
+snmp:
+  community: public
+  timeout: 1
+  retries: 2
+  bulkrows: 35
+dot:
+  bothlinks: true
+  color:
+  - - cpe
+    - gold
+  - - -sw
+    - blue
+  - - -pe
+    - red
+  - - ' -p'
+    - yellow
+dns:
+  afi: 
+log: STDERR
+debug: false
+namemap:
+- - -re\d+
+  - ''
+- - (.*(?<!as23456.net)$)
+  - \1.as23456.net
+```
+
+  * use  - methods to use for crawling
+  * poll - cidrs to allow snmp for
+  * snmp community - snmp community to use
+  * snmp timout    - snmp timout in seconds
+  * snmp retries   - snmp retries count
+  * snmp bulkrows  - snmp row count for bulkget
+  * dot bothlinks  - show a-b and b-a link
+  * dot color      - regexp to color, first hit used
+  * dns afi        - ipv4/ipv6 or nil
+  * log            - STDERR/STDOUT or file
+  * debug          - debugging
+  * namemap        - map (LLDP) name to FQDN (JunOS does not give domain)
+
+## Library use
+require 'netcrawl'
+output = NetCrawl.new.crawl('192.0.2.1').to_hash

data/Rakefile

@@ -0,0 +1,47 @@
+begin
+  require 'rake/testtask'
+  require 'bundler'
+  Bundler.setup
+rescue LoadError
+  warn 'bunler missing'
+  exit 42
+end
+
+gemspec = eval(File.read(Dir['*.gemspec'].first))
+file    = [gemspec.name, gemspec.version].join('-') + '.gem'
+
+desc 'Validate gemspec'
+task :gemspec do
+  gemspec.validate
+end
+
+desc 'Run minitest'
+task :test do
+  Rake::TestTask.new do |t|
+    t.libs.push "lib"
+    t.test_files = FileList['spec/*_spec.rb']
+    t.verbose = true
+  end
+end
+
+desc 'Build gem'
+task :build do
+  system "gem build #{gemspec.name}.gemspec"
+  FileUtils.mkdir_p 'gems'
+  FileUtils.mv file, 'gems'
+end
+
+desc 'Install gem'
+task :install => :build do
+  system "sudo -E sh -c \'umask 022; gem install gems/#{file}\'"
+end
+
+desc 'Remove gems'
+task :clean do
+  FileUtils.rm_rf 'gems'
+end
+
+desc 'Push to rubygems'
+task :push do
+  system "gem push gems/#{file}"
+end

data/bin/netcrawl

@@ -0,0 +1,9 @@
+#!/usr/bin/env ruby
+
+begin
+  require 'netcrawl/cli'
+  puts NetCrawl::CLI.new.run
+rescue => error
+  warn "#{error}"
+  raise if NetCrawl::CFG.debug
+end

data/lib/netcrawl.rb

@@ -0,0 +1,63 @@
+require_relative 'netcrawl/config'
+class NetCrawl
+  class NetCrawlError < StandardError; end
+  class MethodNotFound < NetCrawlError; end
+  attr_reader :hosts
+
+  # @param [String] host host to start crawl from
+  # @return [NetCrawl::Output]
+  def crawl host
+    recurse host
+    Output.new @hosts, @poll
+  end
+
+  # @param [String] host host to get list of peers from
+  # @return [Array] list of peers seen connected to host
+  def get host
+    peers = []
+    @methods.each do |method|
+      peers += method.send(:get, host)
+    end
+    peers.uniq
+  end
+
+  # Given string of IP address, recurses through peers seen and populates @hosts hash
+  # @param [String] host host to start recurse from
+  # @return [void]
+  def recurse host
+    peers = get host
+    @hosts[host] = peers
+    peers.each do |peer|
+      next if     @hosts.has_key? peer
+      next unless @poll.include?  peer
+      crawl peer
+    end
+  end
+
+  private
+
+  def initialize
+    @methods = []
+    @hosts   = {}
+    @poll    = PollMap.new
+    CFG.use.each do |method|
+      begin
+        method = File.basename method.to_s
+        file = 'netcrawl/method/' + method.downcase
+        require_relative file
+        @methods.push NetCrawl.const_get(method)
+      rescue NameError, LoadError => error
+        raise MethodNotFound, "unable to find method '#{method}'"
+      end
+    end
+  end
+
+
+end
+require_relative 'netcrawl/pollmap'
+require_relative 'netcrawl/namemap'
+require_relative 'netcrawl/pollmap'
+require_relative 'netcrawl/method/cdp'
+require_relative 'netcrawl/method/lldp'
+require_relative 'netcrawl/dns'
+require_relative 'netcrawl/output'

data/lib/netcrawl/cli.rb

@@ -0,0 +1,59 @@
+require_relative '../netcrawl'
+require 'slop'
+
+class NetCrawl
+  class CLI
+    class MissingHost < NetCrawlError; end
+    class NoConfig < NetCrawlError; end
+
+    def run
+      output = NetCrawl.new.crawl @host
+      output.clean   if @opts[:purge]
+      output.resolve if @opts[:resolve]
+      if @opts[:graphviz]
+        output.to_dot
+      elsif @opts[:list]
+        output.to_list
+      elsif @opts[:json]
+        output.to_json
+      elsif @opts[:yaml]
+        output.to_yaml
+      else
+        output.to_hash
+      end
+    end
+
+    private
+
+    def initialize
+      if Config.system.empty? and Config.user.empty?
+        Config.user = Config.default
+        Config.save :user
+        raise NoConfig, 'edit ~/.config/netcrawl/config'
+      end
+      @opts = opt_parse
+      args  = @opts.parse
+      @host = DNS.getip args.shift
+      CFG.snmp.community = @opts[:community] if @opts[:community]
+      CFG.debug          = true if @opts[:debug]
+      raise MissingHost, 'no hostname given as argument' unless @host
+    end
+
+
+    def opt_parse
+      opts = Slop.parse(:help=>true) do
+        banner 'Usage: netcrawl [options] hostname'
+        on 'g',  'graphviz',  'dot output use \'dot -Tpng -o map.png map.dot\''
+        on 'l',  'list',      'list nodes'
+        on 'j',  'json',      'json output'
+        on 'y',  'yaml',      'yaml output'
+        on 'a',  'hash',      'hash/associative array output'
+        on 'r',  'resolve',   'resolve addresses to names'
+        on 'p',  'purge',     'remove peers not in configured CIDR'
+        on 'c=', 'community', 'SNMP community to use'
+        on 'd',  'debug',     'turn debugging on'
+      end
+    end
+
+  end
+end

data/lib/netcrawl/config.rb

@@ -0,0 +1,37 @@
+require 'asetus'
+require 'logger'
+class NetCrawl
+  Config = Asetus.new :name=>'netcrawl', :load=>false
+  Config.default.use             = %w(LLDP CDP)
+  Config.default.poll            = [  # addresses accepted for polling
+    '192.0.2.0/24',
+    '198.51.100.0/24',
+    '0.0.0.0/0',
+  ]
+  Config.default.snmp.community  = 'public'
+  Config.default.snmp.timeout    = 1
+  Config.default.snmp.retries    = 2
+  Config.default.snmp.bulkrows   = 35    # 1500B packet should fit about 50 :cdpCacheAddress rows
+  Config.default.dot.bothlinks   = false # keep both a-b and b-a links
+  Config.default.dot.color       = [     # regexp of host => color
+    [ 'cpe', 'gold'   ],
+    [ '-sw', 'blue'   ],
+    [ '-pe', 'red'    ],
+    [' -p',  'yellow' ],
+  ]
+  Config.default.dns.afi         = nil # could be 'ipv4' or 'ipv6'
+  Config.default.log             = 'STDERR'
+  Config.default.debug           = false
+  Config.default.namemap         = [ # regexp match+sub of hostname (needed for LLDP)
+    ['-re\d+', ''],
+    ['^KILLME(.*(?<!my.domain.com)$)', '\1.my.domain.com'],  #adds missing domain name
+  ]
+  Config.load
+  CFG = Config.cfg
+  log = CFG.log
+  log = STDERR if log == 'STDERR'
+  log = STDOUT if log == 'STDOUT'
+  Log = Logger.new log
+  Log.level = Logger::INFO
+  Log.level = Logger::DEBUG if CFG.debug
+end

data/lib/netcrawl/dns.rb

@@ -0,0 +1,53 @@
+require 'resolv'
+class NetCrawl
+  class Resolve
+
+    # @param [String] name DNS name which we try to resolve to IP
+    # @return [String, nil] string if name resolves to IP, otherwise nil
+    def getip name
+      if @cacheip.has_key? name
+        @cacheip[name]
+      else
+        ip = nil
+        begin
+          if CFG.dns.afi == 'ipv4'
+            ip = Resolv::DNS.new.getresource(name, Resolv::DNS::Resource::IN::A).address
+          elsif CFG.dns.afi == 'ipv6'
+            ip = Resolv::DNS.new.getresource(name, Resolv::DNS::Resource::IN::AAAA).address
+          else
+            ip = Resolv.getaddress name
+          end
+        rescue => error
+          Log.debug "DNS resolution for '#{name}' raised error '#{error.class}' with message '#{error.message}'"
+          return nil
+        end
+        @cacheip[name] = ip
+      end
+    end
+
+    # @param [String] ip DNS IP which we try to resolve to name
+    # @return [String] name if it resolves, ip otherwise
+    def getname ip
+      if @cachename.has_key? ip
+        @cachename[ip]
+      else
+        name = nil
+        begin
+          name = Resolv.getname ip
+        rescue => error
+          Log.debug "DNS resolution for '#{ip}' raised error '#{error.class}' with message '#{error.message}'"
+          name = ip
+        end
+        @cachename[ip] = name
+      end
+    end
+
+    private
+
+    def initialize
+      @cacheip = {}
+      @cachename = {}
+    end
+  end
+  DNS = Resolve.new
+end

data/lib/netcrawl/method/cdp.rb

@@ -0,0 +1,33 @@
+require_relative '../snmp'
+class NetCrawl
+  class CDP
+    OID = {
+      # http://tools.cisco.com/Support/SNMP/do/BrowseOID.do?local=en&translate=Translate&objectInput=1.3.6.1.4.1.9.9.23.1.2.1.1
+      :cdpCacheAddress  => '1.3.6.1.4.1.9.9.23.1.2.1.1.4',
+      :cdpCacheDeviceId => '1.3.6.1.4.1.9.9.23.1.2.1.1.6',
+    }
+
+    # @param [String] host host to query
+    # @return [Hash] neighbor information
+    def self.get host
+      cdp = new(host)
+      cdp.peers
+    end
+
+    def peers
+      @snmp.bulkwalk(OID[:cdpCacheAddress]).map do |vb|
+        vb.as_ip
+      end
+    rescue SNMP::NoResponse
+      []
+    end
+
+    private
+
+    def initialize host
+      @snmp = SNMP.new host
+    end
+
+
+  end
+end

data/lib/netcrawl/method/lldp.rb

@@ -0,0 +1,35 @@
+require_relative '../snmp'
+class NetCrawl
+  class LLDP
+    include NameMap
+    OID = {
+      # http://standards.ieee.org/getieee802/download/802.1AB-2009.pdf
+      :lldpRemChassisIdSubtype => '1.0.8802.1.1.2.1.4.1.1.4', # CSCO and JNPR use 4 (MAC address) rendering ChassisID useless
+      :lldpRemChassisId        => '1.0.8802.1.1.2.1.4.1.1.5',
+      :lldpRemSysName          => '1.0.8802.1.1.2.1.4.1.1.9',
+    }
+
+    # @param [String] host host to query
+    # @return [Hash] neighbor information
+    def self.get host
+      lldp = new(host)
+      lldp.peers
+    end
+
+    def peers
+      @snmp.bulkwalk(OID[:lldpRemSysName]).map do |vb|
+        DNS.getip namemap(vb.value)
+      end.compact
+    rescue SNMP::NoResponse
+      []
+    end
+
+    private
+
+    def initialize host
+      @snmp = SNMP.new host
+    end
+
+
+  end
+end

data/lib/netcrawl/namemap.rb

@@ -0,0 +1,12 @@
+class NetCrawl
+  module NameMap
+    def namemap name_org
+      name = name_org.dup
+      CFG.namemap.each do |match, replace|
+        re = Regexp.new match
+        name = name.sub re, replace
+      end
+      name
+    end
+  end
+end

data/lib/netcrawl/output.rb

@@ -0,0 +1,72 @@
+class NetCrawl
+  class Output
+    attr_reader :iphash, :namehash, :hash
+
+    # @return [String] pretty print of hash
+    def to_hash
+      require 'pp'
+      out = ''
+      PP.pp @hash, out
+      out
+    end
+
+    # @return [String] yaml of hosts and peers found
+    def to_yaml
+      require 'yaml'
+      YAML.dump @hash
+    end
+
+    # @return [String] json of hosts and peers found
+    def to_json
+      require 'json'
+      JSON.pretty_generate @hash
+    end
+
+    # @return [Array] of nodes found
+    def to_list
+      nodes = []
+      @hash.each do |host, peers|
+        nodes << host
+        nodes << peers
+      end
+      nodes.flatten.uniq.sort
+    end
+
+    # resolves ip addresses and changes @hash to point to the resolved hash
+    # @return [void]
+    def resolve
+      @namehash = {}
+      @iphash.each do |host, peers|
+        host = DNS.getname host
+        @namehash[host] = []
+        peers.each do |peer|
+          @namehash[host].push DNS.getname(peer)
+        end
+      end
+      @hash = @namehash
+    end
+
+    # remove peers not matchin configured CIDR
+    def clean
+      @hash.each do |host, peers|
+        peers = peers.delete_if{|peer|not @pollmap.include? peer}
+      end
+    end
+
+    private
+
+    def initialize hash, pollmap
+      @iphash  = hash
+      @hash    = @iphash
+      @pollmap = pollmap
+    end
+
+    def method_missing name, *args
+      raise NoMethodError, "invalid method #{name} for #{inspect}:#{self.class}" unless name.match(/to_.*/)
+      output = File.basename name[3..-1]
+      require_relative 'output/' + output
+      output = NetCrawl::Output.const_get output.capitalize
+      output.send :output, self
+    end
+  end
+end

data/lib/netcrawl/output/dot.rb

@@ -0,0 +1,53 @@
+class NetCrawl
+  class Output
+    class Dot
+      INDENT = ' ' * 2
+      DEFAULT_COLOR = 'black'
+      def self.output output
+        new(output).to_s
+      end
+
+      def to_s
+        str = "graph NetCrawl {\n"
+        @output.to_list.each do |host|
+          str << INDENT + id(host) + "[color=\"#{color(host)}\"]\n"
+          str << INDENT + id(host) + "[label=\"#{host}\"]\n"
+          if @hash.has_key? host
+            @hash[host].each do |peer|
+              next if not CFG.dot.bothlinks and @connections.include?([peer, host].sort)
+              @connections << [peer, host].sort
+              str << INDENT + INDENT + id(host) + ' -- ' + id(peer) + "\n"
+            end
+          end
+        end
+        str << "}\n"
+        str
+      end
+
+      private
+
+      def initialize output
+        @output      = output
+        @connections = []
+        @hash        = @output.hash
+      end
+
+      def id host
+        host = host.gsub(/[-.]/, '_')
+        '_' + host
+      end
+
+      def color host
+        color = nil
+        CFG.dot.color.each do |re, clr|
+          if host.match re
+            color = clr
+            break
+          end
+        end
+        color or DEFAULT_COLOR
+      end
+    end
+  end
+end
+

data/lib/netcrawl/pollmap.rb

@@ -0,0 +1,13 @@
+require 'ipaddr'
+class NetCrawl
+  class PollMap
+    def initialize
+      @poll = CFG.poll.map do |cidr|
+        IPAddr.new cidr
+      end
+    end
+    def include? addr
+      @poll.any? { |cidr| cidr.include? addr }
+    end
+  end
+end

data/lib/netcrawl/snmp.rb

@@ -0,0 +1,68 @@
+require 'snmp'
+class NetCrawl
+  class SNMP
+    class NoResponse < NetCrawlError; end
+    # Closes the SNMP connection
+    # @return [void]
+    def close
+      @snmp.close
+    end
+
+    # Gets one oid, return value
+    # @param [String] oid to get
+    # @return [SNMP::VarBind]
+    def get oid
+      mget([oid]).first
+    end
+
+    # Get multiple oids, return array of values
+    # @param [Array(String)] oids to get
+    # @return [SNMP::VarBindList]
+    def mget oids
+      snmp :get, oids
+    end
+
+    # Bulkwalk everything below root oid
+    # @param [String] root oid to start from
+    # @return [Array(SNMP::VarBind)]
+    def bulkwalk root
+      last, oid, results = false, root.dup, []
+      root = root.split('.').map{|chr|chr.to_i}
+      while not last
+        vbs = snmp(:get_bulk, 0, CFG.snmp.bulkrows, oid).varbind_list
+        vbs.each do |vb|
+          oid = vb.oid
+          (last = true; break) if not oid[0..root.size-1] == root
+          results.push vb
+        end
+      end
+      results
+    end
+
+    private
+
+    def initialize host, community=CFG.snmp.community, timeout=CFG.snmp.timeout, retries=CFG.snmp.retries
+      @host = host
+      @snmp = ::SNMP::Manager.new :Host=>@host, :Community=>community,
+                                  :Timeout=>timeout, :Retries=>retries,
+                                  :MibModules=>[]
+    end
+
+    def snmp cmd, *args
+      @snmp.send cmd, *args
+    rescue ::SNMP::RequestTimeout, Errno::EACCES => error
+      msg = "host '#{@host}' raised '#{error.class}' with message '#{error.message}' for method '#{cmd}' with args '#{args}'"
+      Log.warn msg
+      raise NoResponse, msg
+    end
+
+  end
+end
+
+module SNMP
+  class VarBind
+    def as_ip
+      SNMP::IpAddress.new(value).to_s
+    end
+  end
+end

data/netcrawl.gemspec

@@ -0,0 +1,18 @@
+Gem::Specification.new do |s|
+  s.name              = 'netcrawl'
+  s.version           = '0.0.1'
+  s.platform          = Gem::Platform::RUBY
+  s.authors           = [ 'Saku Ytti' ]
+  s.email             = %w( saku@ytti.fi )
+  s.homepage          = 'http://github.com/ytti/netcrawl'
+  s.summary           = 'lldp/cdp crawler'
+  s.description       = 'given snmp community and one node crawls through the network to produce list/dot file'
+  s.rubyforge_project = s.name
+  s.files             = `git ls-files`.split("\n")
+  s.executables       = %w( netcrawl )
+  s.require_path      = 'lib'
+
+  s.add_dependency 'snmp'
+  s.add_dependency 'slop'
+  s.add_dependency 'asetus'
+end

metadata

@@ -0,0 +1,119 @@
+--- !ruby/object:Gem::Specification
+name: netcrawl
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+  prerelease: 
+platform: ruby
+authors:
+- Saku Ytti
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2014-03-30 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: snmp
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: slop
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: asetus
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: given snmp community and one node crawls through the network to produce
+  list/dot file
+email:
+- saku@ytti.fi
+executables:
+- netcrawl
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- Gemfile
+- README.md
+- Rakefile
+- bin/netcrawl
+- lib/netcrawl.rb
+- lib/netcrawl/.cli.rb.swp
+- lib/netcrawl/cli.rb
+- lib/netcrawl/config.rb
+- lib/netcrawl/dns.rb
+- lib/netcrawl/method/cdp.rb
+- lib/netcrawl/method/lldp.rb
+- lib/netcrawl/namemap.rb
+- lib/netcrawl/output.rb
+- lib/netcrawl/output/dot.rb
+- lib/netcrawl/pollmap.rb
+- lib/netcrawl/snmp.rb
+- netcrawl.gemspec
+homepage: http://github.com/ytti/netcrawl
+licenses: []
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+      segments:
+      - 0
+      hash: -152408900674429923
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+      segments:
+      - 0
+      hash: -152408900674429923
+requirements: []
+rubyforge_project: netcrawl
+rubygems_version: 1.8.25
+signing_key: 
+specification_version: 3
+summary: lldp/cdp crawler
+test_files: []