irrc 0.1.0 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: d0cea0cd7090c9cdc0050e02cc02368a03e48f12
-  data.tar.gz: 022c5535b1e8acb15e3641fcff65959105ebd5c7
+  metadata.gz: 2153f20f9106d41a4879ffbc6ab3da228fb92c91
+  data.tar.gz: f79d595e7e9feba683de2e366dd08fdc93b9d119
 SHA512:
-  metadata.gz: f3883807ba6822641f396177a1fb187f0d50c96595ac9b4b33c6d2693a789ac720cd7b324aa1b780ee76898760afd247cf93803a0a9d606c668e0bf826bb9f51
-  data.tar.gz: 0e89f38a1c5b90241301ff54839949cc1778dc0194163659f8e60c23b457fe5e29cfbb25806ab41c707dffb9348eb07f29ef27a0230620445076e0d0460f728f
+  metadata.gz: 01730d20bad2b5b2509b1895fc237ebc299d372ba5a053f52cae32b74e31be9580aa3d84856424e946dd1debf0de1c45f7fb8636fff807258fe133f402ad9eb2
+  data.tar.gz: e45bce36c6a1449a6a4f866f6e258e76c264dd96fb6f1bc8eb4be9a1cd9e2badc9f409408fefadc39e8a185783eb6fb8b62415cc3c6f24104c0489923636912e

data/.travis.yml

@@ -1,6 +1,14 @@
 language: ruby
+
 rvm:
-  - 2.0.0
-  - 2.1.0
-  - 2.1.1
-  - ruby-head
+  - 2.1.9
+  - 2.2.6
+  - 2.3.3
+  - ruby-head
+
+before_install:
+  - gem update bundler
+
+matrix:
+  allow_failures:
+    - rvm: ruby-head

data/Gemfile

@@ -1,7 +1,4 @@
 source 'https://rubygems.org'
 
-group :test do
-  gem 'rspec'
-end
-
-gemspec
+# Specify your gem's dependencies in junoser.gemspec
+gemspec

data/LICENSE

@@ -1,6 +1,6 @@
 The MIT License (MIT)
 
-Copyright (c) 2014 Shintaro Kojima
+Copyright (c) 2017 Shintaro Kojima
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

data/README.md

@@ -64,8 +64,8 @@ You can load irrc as a library and use it easily in your own code.
 require 'irrc'
 
 client = Irrc::Client.new
-client.query(:jpirr, 'AS-JPNIC', source: :jpirr)    # queries JPIRR about AS-JPNIC with a SOURCE: filter
-client.query(:ripe, 'AS-RIPENCC', protocol: :ipv4)  # queries RIPE Whoisd about AS-RIPENCC for IPv4 only
+client.query :jpirr, 'AS-JPNIC', source: :jpirr     # queries JPIRR about AS-JPNIC with a SOURCE: filter
+client.query :ripe, 'AS-RIPENCC', protocol: :ipv4   # queries RIPE Whoisd about AS-RIPENCC for IPv4 only
 client.perform                                      # returns the results in a Hash
 ```
 
@@ -99,7 +99,7 @@ AS-JPNIC:               # queried object
 
 * Ruby >= 2.0.0
 
-Successfully tested with 2.0.0, 2.1.0, 2.1.1, 2.1.2.
+Successfully tested with 2.1.9, 2.2.6, 2.3.3 and 2.4.0.
 
 
 ## Threading
@@ -146,34 +146,12 @@ client = Irrc::Client.new {|c| c.logger = Rails.logger }
 
 Here is a quick performance comparison with peval and irrpt.
 
-* Ordinary as-set
-
-| CLI command                     |      user |     system |     cpu |      total |
-| :------------------------------ | --------: | ---------: | ------: | ---------: |
-| peval -h jpirr.nic.ad.jp AS-OCN |     0.14s |      0.04s |      1% |      9.397 |
-| irrpt_eval AS-OCN               |     0.35s |      0.06s |      5% |      8.021 |
-| **irrc -h jpirr AS-OCN**        | **0.42s** |  **0.09s** |  **4%** | **10.639** |
-
-* Huge as-set
-
-Object caching may contribute to the query performance.
-
-| CLI command                               |        user |     system |     cpu |       total |
-| :---------------------------------------- | ----------: | ---------: | ------: | ----------: |
-| peval -h jpirr.nic.ad.jp AS-HURRICANE     |      67.29s |      1.21s |     22% |     4:58.16 |
-| irrpt_eval AS-HURRICANE                   |     163.10s |      1.38s |     43% |     6:18.17 |
-| **irrc -h jpirr AS-HURRICANE**            |  **20.26s** |  **1.84s** |  **7%** | **4:54.37** |
-
-* Multiple as-set
-
-Multi-threading and object caching may contribute to the query performance.
-
-| CLI command                                                    |        user |     system |     cpu |       total |
-| :------------------------------------------------------------- | ----------: | ---------: | ------: | ----------: |
-| for i in AS-HURRICANE AS-GBLX; peval -h jpirr.nic.ad.jp $i     |      76.47s |      1.08s |     20% |     6:15.80 |
-| irrpt_fetch                                                    |     174.21s |      1.29s |     44% |     6:38.50 |
-| irrc -h jpirr AS-HURRICANE AS-GBLX                             |      22.28s |      1.75s |      8% |     4:34.31 |
-| **irrc -h jpirr -t 2 AS-HURRICANE AS-GBLX**                    |  **23.04s** |  **1.81s** |  **9%** | **4:09.27** |
+| CLI command                                      |      user |     system |     cpu |      total |
+| :----------------------------------------------- | --------: | ---------: | ------: | ---------: |
+| peval -h jpirr.nic.ad.jp 'afi ipv4, ipv6 AS-OCN' |     0.15s |      0.04s |      3% |      4.959 |
+| irrpt_list_prefixes AS-OCN                       |     0.21s |      0.09s |      3% |      9.693 |
+| **irrc -h jpirr AS-OCN**                         | **0.42s** |  **0.12s** |  **5%** |  **9.622** |
+| **irrc -h jpirr -t 4 AS-OCN**                    | **0.39s** |  **0.13s** | **19%** |  **2.754** |
 
 
 ## Contributing
@@ -192,4 +170,4 @@ For questions or feedbacks write to my twitter @codeout.
 
 ## Copyright and License
 
-Copyright (c) 2014 Shintaro Kojima. Code released under the [MIT license](LICENSE).
+Copyright (c) 2017 Shintaro Kojima. Code released under the [MIT license](LICENSE).

data/bin/console

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+
+require "bundler/setup"
+require "foo"
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+# require "pry"
+# Pry.start
+
+require "irb"
+IRB.start

data/bin/setup

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+set -vx
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/exe/irrc

@@ -0,0 +1,89 @@
+#!/usr/bin/env ruby
+
+require 'optparse'
+require 'yaml'
+
+require 'irrc'
+
+
+options = Struct.new(:host, :source, :protocol, :threads, :debug).new(nil, [], [], 1, nil)
+
+opts = OptionParser.new do |opts|
+  opts.banner = <<-EOS
+
+Usage: #{opts.program_name} [options] object [object ...]
+
+       If no [-4|-6|--ipv4|--ipv6] option given, it tries both of ipv4 and ipv6.
+  EOS
+  opts.separator ''
+  opts.separator 'Options:'
+
+  opts.on '-h HOST', 'Specify FQDN of IRR / Whois server to send queries.',
+          'IRR / Whois name is also acceptable. This switch is mandatory.' do |host|
+    options.host = host
+  end
+
+  opts.on '-s SOURCE', '--source', 'Specify an authoritative IRR / Whois source name.',
+          'Multiply this option for multiple SOURCE.',
+          "eg) #{opts.program_name} -s jpirr -s radb AS-JPNIC" do |source|
+    options.source |= [source]
+  end
+
+  opts.on '-4', '--ipv4', 'Resolve IPv4 prefixes.' do
+    options.protocol |= [:ipv4]
+  end
+
+  opts.on '-6', '--ipv6', 'Resolve IPv6 prefixes.' do
+    options.protocol |= [:ipv6]
+  end
+
+  opts.on '-t NUMBER', '--threads', 'Number of threads to resolve prefixes per IRR / Whois server.' do |threads|
+    options.threads = threads
+  end
+
+  opts.on '-d', '--debug', 'Print raw queries, answers and additional informations.' do
+    options.debug = true
+  end
+
+  opts.on_tail '--help', 'Show this message' do
+    puts "irrc: Better IRR client to resolve as-set, route-set or aut-num object into prefixes", opts
+    exit
+  end
+
+  opts.on_tail '-v', '--version', 'Show version' do
+    puts Irrc::VERSION
+    exit
+  end
+end
+opts.parse!
+
+
+if ARGV.empty?
+  $stderr.puts 'Missing Argument: object is required.', opts
+  abort
+end
+
+unless options.host
+  $stderr.puts 'Missing Argument: -h option is required.', opts
+  abort
+end
+
+
+options.protocol = [:ipv4, :ipv6] if options.protocol.empty?
+
+
+begin
+  client = if options.debug
+             Irrc::Client.new(options.threads) {|c| c.logger = Irrc::Logging::Logger.new(STDERR) }
+           else
+             Irrc::Client.new(options.threads)
+           end
+
+  client.query options.host, ARGV, source: options.source, protocol: options.protocol
+  puts client.perform.to_yaml
+rescue
+  $stderr.print "#{$!.class}: " unless $!.instance_of?(RuntimeError)
+  $stderr.puts $!.message
+
+  abort
+end

data/irrc.gemspec

@@ -8,16 +8,21 @@ Gem::Specification.new do |spec|
   spec.version       = Irrc::VERSION
   spec.authors       = ['Shintaro Kojima']
   spec.email         = ['goodies@codeout.net']
+
   spec.summary       = 'IRR / Whois client to expand as-set and route-set into a list of origin ASs and prefixes'
   spec.description   = 'irrc is a lightweight and flexible client of IRR / Whois Database to expand arbitrary as-set and route-set objects into a list of origin ASs and prefixes belonging to the ASs. It allows concurrent queries to IRR / Whois Databases for performance.'
   spec.homepage      = 'https://github.com/codeout/irrc'
   spec.license       = 'MIT'
 
-  spec.files         = `git ls-files -z`.split("\x0")
-  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
-  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
-  spec.require_paths = ['lib']
+  spec.files         = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
+  spec.bindir        = "exe"
+  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.require_paths = ["lib"]
+
+  spec.add_dependency 'net-telnet'
 
-  spec.add_development_dependency 'bundler', '~> 1.6'
-  spec.add_development_dependency 'rake'
+  spec.add_development_dependency 'bundler', '~> 1.9'
+  spec.add_development_dependency 'rake', '~> 10.0'
+  spec.add_development_dependency 'rspec'
+  spec.required_ruby_version = '>= 2.0.0'
 end

data/lib/irrc/client.rb

@@ -37,15 +37,14 @@ module Irrc
     #
     # Examples
     #
-    #   client.query(:jpirr, 'AS-JPNIC', source: :jpirr, protocol: :ipv4)
-    #   client.query(:jpirr, 'AS-JPNIC', source: [:jpirr, :radb])
+    #   client.query :jpirr, 'AS-JPNIC', source: :jpirr, protocol: :ipv4
+    #   client.query :jpirr, 'AS-JPNIC', source: [:jpirr, :radb]
     def query(host, objects, options={})
-      raise ArgumentError, 'host required.' unless host
+      raise ArgumentError, 'host is required.' unless host
       fqdn = Irrc::Irr.host(host) || host
 
-      queues[fqdn] ||= Queue.new
       Array(objects).map {|object|
-        queues[fqdn] << Irrc::Query.new(object, options)
+        queue(fqdn) << Irrc::Query.new(object, options)
       }
     end
 
@@ -56,9 +55,10 @@ module Irrc
       done = []
 
       queues.each_with_object([]) {|(fqdn, queue), workers|
-        @thread_limit.times.map {
+        @thread_limit.times.map.with_index {|i|
           workers << Thread.start {
-            done.push *worker_class(fqdn).new(fqdn, queues[fqdn], &@block).run
+            Thread.current[:id] = i+1
+            done.push *worker_class(fqdn).new(fqdn, queue(fqdn), cache(fqdn), &@block).run(@thread_limit)
           }
         }
       }.each {|t| t.join }
@@ -90,7 +90,19 @@ module Irrc
     private
 
     def queues
-      @_queues ||= {}
+      @queues ||= {}
+    end
+
+    def queue(fqdn)
+      queues[fqdn] ||= Queue.new
+    end
+
+    def caches
+      @caches ||= {}
+    end
+
+    def cache(fqdn)
+      caches[fqdn] ||= {}
     end
 
     def worker_class(fqdn)

data/lib/irrc/connecting.rb

@@ -11,8 +11,8 @@ module Irrc
     end
 
     def connect
-      @connection ||= logger.info("Connecting to #{@host}") &&
-        Net::Telnet.new('Host' => @host,
+      @connection ||= logger.info("Connecting to #{@fqdn}") &&
+        Net::Telnet.new('Host' => @fqdn,
         'Port' => 43,
         'Telnetmode' => false,
         'Prompt' => return_code)
@@ -20,7 +20,7 @@ module Irrc
 
     def close
       if established?
-        logger.info "Closing a connection to #{@host}"
+        logger.info "Closing a connection to #{@fqdn}"
         @connection.close
       end
     end
@@ -28,12 +28,5 @@ module Irrc
     def established?
       @connection && !@connection.sock.closed?
     end
-
-    def execute(command)
-      return if command.nil? || command == ''
-
-      logger.debug "Executing: #{command}"
-      @connection.cmd(command).tap {|result| logger.debug "Returned: #{result}" }
-    end
   end
 end

data/lib/irrc/irr.rb

@@ -48,49 +48,43 @@ module Irrc
 
       # See http://www.irr.net/docs/list.html
       LIST = [
-        ['ALTDB',        'whois.altdb.net',         'irrd'],
-        ['AOLTW',        'whois.aoltw.net',         'irrd'],
-        ['APNIC',        'whois.apnic.net',         'whoisd'],
-        ['ARIN',         'rr.arin.net',             'whoisd'],
-#        ['BCNET',        'whois.bc.net',            nil],
-        ['BELL',         'whois.in.bell.ca',        'irrd'],
-        ['BBOI',         'irr.bboi.net',            'irrd'],
-        ['CANARIE',      'whois.canarie.ca',        'whoisd'],
-        ['D',            'whois.depository.net',    'whoisd'],
-#        ['DERU',         'whois.deru.net',          nil],
-#        ['DIGITALREALM', 'rr.digitalrealm.net',     nil],
-        ['EASYNET',      'whois.noc.easynet.net',   'whoisd'],
-#        ['EBIT',         'whois.ebit.ca',           nil],
-        ['EPOCH',        'whois.epoch.net',         'irrd'],
-        ['GT',           'rr.gt.ca',                'irrd'],
-#        ['GW',           'whois.gw.net',            nil],
-        ['HOST',         'rr.host.net',             'irrd'],
-        ['JPIRR',        'jpirr.nic.ad.jp',         'irrd'],
-        ['LEVEL3',       'rr.level3.net',           'whoisd'],
-#        ['MTO',          'rr.mtotelecom.com',       nil],
-        ['NESTEGG',      'whois.nestegg.net',       'irrd'],
-        ['NTTCOM',       'rr.ntt.net',              'irrd'],
-        ['OPENFACE',     'whois.openface.ca',       'irrd'],
-        ['OTTIX',        'whois.ottix.net',         'irrd'],
-        ['PANIX',        'rrdb.access.net',         'irrd'],
-        ['RADB',         'whois.radb.net',          'irrd'],
-        ['REACH',        'rr.net.reach.com',        'irrd'],
-        ['RGNET',        'whois.rg.net',            'irrd'],
-        ['RIPE',         'whois.ripe.net',          'whoisd'],
-        ['RISQ',         'rr.risq.net',             'irrd'],
-        ['ROGERS',       'whois.rogerstelecom.net', 'irrd'],
-        ['SAVVIS',       'rr.savvis.net',           'whoisd'],
-        ['TC',           'bgp.net.br',              'irrd']
+        ['AFRINIC',  'whois.afrinic.net',       'whoisd'],
+        ['ALTDB',    'whois.altdb.net',         'irrd'],
+        ['AOLTW',    'whois.aoltw.net',         'irrd'],
+        ['APNIC',    'whois.apnic.net',         'whoisd'],
+        ['ARIN',     'rr.arin.net',             'whoisd'],
+        ['BELL',     'whois.in.bell.ca',        'irrd'],
+        ['BBOI',     'irr.bboi.net',            'irrd'],
+        ['CANARIE',  'whois.canarie.ca',        'whoisd'],
+        ['EASYNET',  'whois.noc.easynet.net',   'whoisd'],
+        ['EPOCH',    'whois.epoch.net',         'irrd'],
+        ['GT',       'rr.gt.ca',                'irrd'],
+        ['HOST',     'rr.host.net',             'irrd'],
+        ['JPIRR',    'jpirr.nic.ad.jp',         'irrd'],
+        ['LEVEL3',   'rr.level3.net',           'whoisd'],
+        ['NESTEGG',  'whois.nestegg.net',       'irrd'],
+        ['NTTCOM',   'rr.ntt.net',              'irrd'],
+        ['OPENFACE', 'whois.openface.ca',       'irrd'],
+        ['OTTIX',    'whois.ottix.net',         'irrd'],
+        ['PANIX',    'rrdb.access.net',         'irrd'],
+        ['RADB',     'whois.radb.net',          'irrd'],
+        ['REACH',    'rr.net.reach.com',        'irrd'],
+        ['RGNET',    'whois.rg.net',            'irrd'],
+        ['RIPE',     'whois.ripe.net',          'whoisd'],
+        ['RISQ',     'rr.risq.net',             'irrd'],
+        ['ROGERS',   'whois.rogerstelecom.net', 'irrd'],
+        ['SAVVIS',   'rr.savvis.net',           'whoisd'],
+        ['TC',       'whois.bgp.net.br',        'irrd']
       ]
     end
 
 
     def object
-      @_object
+      @object
     end
 
     def object=(object)
-      @_object = object
+      @object = object
     end
 
     # Public: Returns the object type to query.
@@ -98,7 +92,7 @@ module Irrc
     #
     # Returns: A String. ('as-set', 'route-set' or 'aut-num')
     def object_type
-      case @_object
+      case @object
       when /^AS-[\w-]+$|:AS-[\w-]+$/i
         'as-set'
       when /^RS-[\w-]+$|:RS-[\w-]+$/i

data/lib/irrc/irrd/client.rb

@@ -20,8 +20,6 @@ module Irrc
       include Irrc::Runner
       include Irrc::Irrd::Api
 
-      attr_reader :host, :queue
-
 
       private
 
@@ -35,54 +33,73 @@ module Irrc
 
         case query.object_type
         when 'as-set'
-          resolve_aut_nums_from_as_set query
-          resolve_prefixes_from_aut_nums query
+          expand_as_set query
         when 'route-set'
-          resolve_prefixes_from_route_set query
+          expand_route_set query
         when 'aut-num'
-          query.add_aut_num_result query.object
-          resolve_prefixes_from_aut_nums query
+          expand_aut_num query
         end
+
+        query
       end
 
       def set_source(query)
         command = set_source_command(query.sources)
         if execute(command) =~ error_code
-          raise "'#{command}' failed on '#{host}' (#{$1})."
+          raise "'#{command}' failed on '#{@fqdn}' (#{$1})."
         end
       end
 
-      def resolve_aut_nums_from_as_set(query)
-        command = expand_set_command(query.object)
-        result = execute(command)
-        query.add_aut_num_result parse_aut_nums_from_as_set(result)
-      rescue
-        raise "'#{command}' failed on '#{host}' (#{$!.message})."
+      # Public: Expand an as-set object into aut-nums
+      def expand_as_set(query)
+        result = cache(query.object, query.sources) {
+          begin
+            command = expand_set_command(query.object)
+            execute(command)
+          rescue
+            raise "'#{command}' failed on '#{@fqdn}' (#{$!.message})."
+          end
+        }
+
+        parse_aut_nums_from_as_set(result).each do |autnum|
+          child = query.fork(autnum)
+          query.add_aut_num_result autnum if child.aut_num?
+        end
       end
 
-      def resolve_prefixes_from_route_set(query)
-        command = expand_set_command(query.object)
-        result = execute(command)
+      # Public: Expand a route-set into routes
+      def expand_route_set(query)
+        result = cache(query.object, query.sources) {
+          begin
+            command = expand_set_command(query.object)
+            execute(command)
+          rescue
+            raise "'#{command}' failed on '#{@fqdn}' (#{$!.message})."
+          end
+        }
+
         prefixes = classify_by_protocol(parse_prefixes_from_route_set(result))
 
         query.protocols.each do |protocol|
           query.add_prefix_result prefixes[protocol], nil, protocol
         end
-      rescue
-        raise "'#{command}' failed on '#{host}' (#{$!.message})."
       end
 
-      def resolve_prefixes_from_aut_nums(query)
-        unless query.protocols.empty?
-          # ipv4 and ipv6 should have the same result so far
-          (query.result[:ipv4] || query.result[:ipv6]).keys.each do |autnum|
-            command = expand_aut_num_command(autnum)
-            result = execute(command)
+      # Public: Expand an aut-num object into routes
+      def expand_aut_num(query)
+        return if query.protocols.empty?
 
-            query.protocols.each do |protocol|
-              query.add_prefix_result parse_prefixes_from_aut_num(result, protocol), autnum, protocol
-            end
+        result = cache(query.object, query.sources) {
+          begin
+            command = expand_aut_num_command(query.object)
+            execute(command)
+          rescue
+            raise "'#{command}' failed on '#{@fqdn}' (#{$!.message})."
           end
+        }
+
+        query.protocols.each do |protocol|
+          query.add_prefix_result parse_prefixes_from_aut_num(result, protocol), query.object, protocol
         end
       end
     end