passivedns-client 2.1.12 → 2.1.13

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: d8ea3feed82ed9f049ef8d7cd6ac78c087b94a630449cdbb3560fc28364ffe41
-  data.tar.gz: 37056ad39c2d721a3b5e6ee9f0347c5f1f79686f75eb15da99e2f155fc7b22d0
+  metadata.gz: 8c2b55f21b59fb20d9167e1893d86aa500e3d020932d7b6e3fe5f2eed7e3113b
+  data.tar.gz: 7a48256ee4739a8a432a4a5f7aff6c7a9d8fcd1e5922b07caaf6d3557ffa5ca5
 SHA512:
-  metadata.gz: 61ca76c52991e1a9e131d51412f0a855301b9555cbd0d1d6e5fe9c0071c1bc2d0b1539926148c090ad31c7db1aaff544565537c4db2f59bfd449e6747786d655
-  data.tar.gz: 4bae5d3265ca1723c9170dc3e1ec726a6d855ff2324b4744692f0a9a486444751723fa907ce50ef9960900fa4236b53fdfb1484f04fc4a91c7dc1e6d61889756
+  metadata.gz: e75a301a616818187ba3ea036b2ae39b8b3b562c37115cf4b6d1ece009748838d76477ef58b353f654a81e890c1f5aaa2aeb151b45c23cb5bec692845f52f732
+  data.tar.gz: 6a549456ebbc19920f00129ee0371742790f288324ade26441349cd407d9e7f8f42e49ecaac49da3b59ed5ba36267fb925d77cfc33e996d58df4d8284a28f5d2

data/README.md

@@ -2,14 +2,10 @@
 
 This rubygem queries the following Passive DNS databases: 
 
-* BFK.de
 * CIRCL
 * DNSDB (FarSight)
-* Mnemonic
-* PassiveDNS.cn (Qihoo 360 Technology Co.,Ltd)
 * PassiveTotal
 * RiskIQ
-* TCPIPUtils
 * VirusTotal
 
 Passive DNS is a technique where IP to hostname mappings are made by recording the answers of other people's queries.  
@@ -40,16 +36,8 @@ From version 2.0.0 on, all configuration keys for passive DNS providers are in o
 
 	[dnsdb]
 	APIKEY = 0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef
-	[cn360]
-	API = http://some.web.address.for.their.api
-	API_ID = a username that is given when you register
-	API_KEY = a long and random password of sorts that is used along with the page request to generate a per page API key
-	[tcpiputils]
-	APIKEY = 0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef
 	[virustotal]
 	APIKEY = 0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef
-	[mnemonic]
-	APIKEY = 01234567890abcdef01234567890abcdef012345
 	[passivetotal]
 	USERNAME = tom@example.com
 	APIKEY = 0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef
@@ -66,61 +54,55 @@ CIRCL also can use and authorization token.  In that case, you should drop the U
 	AUTH_TOKEN = 0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef
 
 ## Getting Access
-* 360.cn : http://www.passivedns.cn
-* BFK.de : No registration required, but please, please ready their usage policy at http://www.bfk.de/bfk_dnslogger.html
 * CIRCL : https://www.circl.lu/services/passive-dns/
 * DNSDB (Farsight Security) : https://api.dnsdb.info/
-* Mnemonic : mss .at. mnemonic.no
 * PassiveTotal : https://www.passivetotal.org
 * RiskIQ : https://github.com/RiskIQ/python_api/blob/master/LICENSE
-* TCPIPUtils : http://www.tcpiputils.com/premium-access
 * VirusTotal : https://www.virustotal.com
 
 ## Usage
 
 	require 'passivedns/client'
 	
-	c = PassiveDNS::Client.new(['bfk','dnsdb']) # providers: bfk, tcpiputils, certee, dnsdb, virustotal, passivedns.cn, mnemonic
+	c = PassiveDNS::Client.new(['riskiq','dnsdb'])
 	results = c.query("example.com")
 	
 
 Or use the included tool...
 
-	Usage: bin/pdnstool [-d [3bcdmptv]] [-g|-v|-m|-c|-x|-y|-j|-t] [-os <sep>] [-f <file>] [-r#|-w#|-v] [-l <count>] <ip|domain|cidr>
-	Passive DNS Providers  -d3bcdmptv uses all of the available passive dns database
-	  -d3 use 360.cn
-	  -db use BFK.de
-	  -dc use CIRCL
-	  -dd use DNSDB
-	  -dm use Mnemonic
-	  -dp use PassiveTotal
-	  -dr use RiskIQ
-	  -dt use TCPIPUtils
-	  -dv use VirusTotal
-	  -dvt uses VirusTotal and TCPIPUtils (for example)
-	  
-	Output Formatting
-	  -g link-nodal GDF visualization definition
-	  -z link-nodal graphviz visualization definition
-	  -m link-nodal graphml visualization definition
-	  -c CSV
-	  -x XML
-	  -y YAML
-	  -j JSON
-	  -t ASCII text (default)
-	  -s <sep> specifies a field separator for text output, default is tab
-	  
-	State and Recusion
-	  -f[file] specifies a sqlite3 database used to read the current state - useful for large result sets and generating graphs of previous runs.
-	  -r# specifies the levels of recursion to pull. **WARNING** This is quite taxing on the pDNS servers, so use judiciously (never more than 3 or so) or find yourself blocked!
-	  -w# specifies the amount of time to wait, in seconds, between queries (Default: 0)
-	  -l <count> limits the number of records returned per passive dns database queried.
-    
-	Specifying a Configuration File
-	  --config <file> specifies a config file. default: #{ENV['HOME']}/.passivedns-client
-	  
-	Getting Help
-	  -v debugging information
+    Usage: bin/pdnstool [-d [cdprv]] [-g|-v|-m|-c|-x|-y|-j|-t] [-os <sep>] [-f <file>] [-r#|-w#|-v] [-l <count>] [--config <file>] <ip|domain|cidr>
+    Passive DNS Providers
+      -dcdprv uses all of the available passive dns database
+      -dc use CIRCL
+      -dd use DNSDB
+      -dp use PassiveTotal
+      -dr use RiskIQ
+      -dv use VirusTotal
+      -dvr uses VirusTotal and RiskIQ (for example)
+
+    Output Formatting
+      -g link-nodal GDF visualization definition
+      -z link-nodal graphviz visualization definition
+      -m link-nodal graphml visualization definition
+      -c CSV
+      -x XML
+      -y YAML
+      -j JSON
+      -t ASCII text (default)
+      -s <sep> specifies a field separator for text output, default is tab
+
+    State and Recursion
+      -f[file] specifies a sqlite3 database used to read the current state - useful for large result sets and generating graphs of previous runs.
+      -r# specifies the levels of recursion to pull. **WARNING** This is quite taxing on the pDNS servers, so use judiciously (never more than 3 or so) or find yourself blocked!
+      -w# specifies the amount of time to wait, in seconds, between queries (Default: 0)
+      -l <count> limits the number of records returned per passive dns database queried.
+
+    Specifying a Configuration File
+      --config <file> specifies a config file. default: /home/chris/.passivedns-client
+
+    Getting Help
+      -h hello there.  This option produces this helpful help information on how to access help.
+      -v debugging information
 
 ## Writing Your Own Database Adaptor
 

data/lib/passivedns/client.rb

@@ -12,9 +12,7 @@ provider_path = File.dirname(__FILE__)+"/client/provider/*.rb"
 Dir.glob(provider_path).each do |provider|
   name = File.basename(provider, '.rb')
   require "passivedns/client/provider/#{name}.rb"
-  if name != 'bfk'
-    $passivedns_providers << name
-  end
+  $passivedns_providers << name
 end
 
 require 'configparser'

data/lib/passivedns/client/cli.rb

@@ -133,12 +133,7 @@ module PassiveDNS # :nodoc:
       ARGV.replace(origARGV)
 
       if options[:pdnsdbs].length == 0
-        options[:pdnsdbs] << "bfk"
-      end
-
-      if options[:pdnsdbs].index("bfk") and options[:recursedepth] > 1 and options[:wait] < 60
-        options[:wait] = 60
-        $stderr.puts "Enforcing a minimal 60 second wait when using BFK for recursive crawling"
+        options[:pdnsdbs] << "virustotal"
       end
 
       if options[:debug]
@@ -168,7 +163,7 @@ module PassiveDNS # :nodoc:
       letter_map.keys.sort.each do |l|
         help_text << "  -d#{l} use #{letter_map[l][0]}\n"
       end
-      help_text << "  -dvt uses VirusTotal and TCPIPUtils (for example)\n"
+      help_text << "  -dvr uses VirusTotal and RiskIQ (for example)\n"
       help_text << "\n"
       help_text << "Output Formatting\n"
       help_text << "  -g link-nodal GDF visualization definition\n"

data/lib/passivedns/client/version.rb

@@ -2,6 +2,6 @@ module PassiveDNS # :nodoc:
   # coodinates the lookups accross all configured PassiveDNS providers
   class Client
     # version of PassiveDNS::Client
-    VERSION = "2.1.12"
+    VERSION = "2.1.13"
   end
 end

data/test/test_cli.rb

@@ -13,26 +13,22 @@ require_relative '../lib/passivedns/client/cli.rb'
 class TestCLI < Minitest::Test
   def test_letter_map
     letter_map = PassiveDNS::CLI.get_letter_map
-    assert_equal("3bcdmprtv", letter_map.keys.sort.join(""))
+    assert_equal("cdprv", letter_map.keys.sort.join(""))
   end
   
   def test_help_text
     helptext = PassiveDNS::CLI.run(["--help"])
     helptext.gsub!(/Usage: .*?\[/, "Usage: [")
     assert_equal(
-"Usage: [-d [3bcdmprtv]] [-g|-v|-m|-c|-x|-y|-j|-t] [-os <sep>] [-f <file>] [-r#|-w#|-v] [-l <count>] [--config <file>] <ip|domain|cidr>
+"Usage: [-d [cdprv]] [-g|-v|-m|-c|-x|-y|-j|-t] [-os <sep>] [-f <file>] [-r#|-w#|-v] [-l <count>] [--config <file>] <ip|domain|cidr>
 Passive DNS Providers
-  -d3bcdmprtv uses all of the available passive dns database
-  -d3 use 360.cn
-  -db use BFK.de
+  -dcdprv uses all of the available passive dns database
   -dc use CIRCL
   -dd use DNSDB
-  -dm use Mnemonic
   -dp use PassiveTotal
   -dr use RiskIQ
-  -dt use TCPIPUtils
   -dv use VirusTotal
-  -dvt uses VirusTotal and TCPIPUtils (for example)
+  -dvr uses VirusTotal and RiskIQ (for example)
 
 Output Formatting
   -g link-nodal GDF visualization definition
@@ -62,7 +58,7 @@ Getting Help
   
   def test_provider_parsing
     options_target = {
-      :pdnsdbs => ["bfk"],
+      :pdnsdbs => ["virustotal"],
       :format => "text",
       :sep => "\t",
       :recursedepth => 1,
@@ -78,24 +74,14 @@ Getting Help
     options, items = PassiveDNS::CLI.parse_command_line([])
     assert_equal(options_target, options)
     assert_equal([], items)
-    
-    options_target[:pdnsdbs] = ["cn360"]
-    options, items = PassiveDNS::CLI.parse_command_line(["-d3"])
-    assert_equal(options_target, options)
-    assert_equal([], items)
-
-    options_target[:pdnsdbs] = ["bfk"]
-    options, items = PassiveDNS::CLI.parse_command_line(["-db"])
-    assert_equal(options_target, options)
-    assert_equal([], items)
-   
-    options_target[:pdnsdbs] = ["circl", "dnsdb", "mnemonic", "riskiq"]
-    options, items = PassiveDNS::CLI.parse_command_line(["-dcdmr"])
+       
+    options_target[:pdnsdbs] = ["circl", "dnsdb", "riskiq"]
+    options, items = PassiveDNS::CLI.parse_command_line(["-dcdr"])
     assert_equal(options_target, options)
     assert_equal([], items)
     
-    options_target[:pdnsdbs] = ["passivetotal", "tcpiputils", "virustotal"]
-    options, items = PassiveDNS::CLI.parse_command_line(["-dptv"])
+    options_target[:pdnsdbs] = ["passivetotal", "virustotal"]
+    options, items = PassiveDNS::CLI.parse_command_line(["-dpv"])
     assert_equal(options_target, options)
     assert_equal([], items)
     
@@ -103,7 +89,7 @@ Getting Help
   
   def test_output_parsing
     options_target = {
-      :pdnsdbs => ["passivetotal", "tcpiputils", "virustotal"],
+      :pdnsdbs => ["passivetotal", "virustotal"],
       :format => "text",
       :sep => "\t",
       :recursedepth => 1,
@@ -117,48 +103,48 @@ Getting Help
     }
     
     options_target[:sep] = ","
-    options, items = PassiveDNS::CLI.parse_command_line(["-dptv", "-c", "8.8.8.8"])
+    options, items = PassiveDNS::CLI.parse_command_line(["-dpv", "-c", "8.8.8.8"])
     assert_equal(options_target, options)
     assert_equal(["8.8.8.8"], items)
     
     options_target[:sep] = "|"
-    options, items = PassiveDNS::CLI.parse_command_line(["-dptv", "-s", "|", "8.8.8.8"])
+    options, items = PassiveDNS::CLI.parse_command_line(["-dpv", "-s", "|", "8.8.8.8"])
     assert_equal(options_target, options)
     assert_equal(["8.8.8.8"], items)
     
     options_target[:sep] = "\t"
     
-    options, items = PassiveDNS::CLI.parse_command_line(["-dptv", "-t", "8.8.8.8"])
+    options, items = PassiveDNS::CLI.parse_command_line(["-dpv", "-t", "8.8.8.8"])
     assert_equal(options_target, options)
     assert_equal(["8.8.8.8"], items)
 
     options_target[:format] = "json"
-    options, items = PassiveDNS::CLI.parse_command_line(["-dptv", "-j", "8.8.8.8"])
+    options, items = PassiveDNS::CLI.parse_command_line(["-dpv", "-j", "8.8.8.8"])
     assert_equal(options_target, options)
     assert_equal(["8.8.8.8"], items)
     
     options_target[:format] = "xml"
-    options, items = PassiveDNS::CLI.parse_command_line(["-dptv", "-x", "8.8.8.8"])
+    options, items = PassiveDNS::CLI.parse_command_line(["-dpv", "-x", "8.8.8.8"])
     assert_equal(options_target, options)
     assert_equal(["8.8.8.8"], items)
     
     options_target[:format] = "yaml"
-    options, items = PassiveDNS::CLI.parse_command_line(["-dptv", "-y", "8.8.8.8"])
+    options, items = PassiveDNS::CLI.parse_command_line(["-dpv", "-y", "8.8.8.8"])
     assert_equal(options_target, options)
     assert_equal(["8.8.8.8"], items)
     
     options_target[:format] = "gdf"
-    options, items = PassiveDNS::CLI.parse_command_line(["-dptv", "-g", "8.8.8.8"])
+    options, items = PassiveDNS::CLI.parse_command_line(["-dpv", "-g", "8.8.8.8"])
     assert_equal(options_target, options)
     assert_equal(["8.8.8.8"], items)
     
     options_target[:format] = "graphviz"
-    options, items = PassiveDNS::CLI.parse_command_line(["-dptv", "-z", "8.8.8.8"])
+    options, items = PassiveDNS::CLI.parse_command_line(["-dpv", "-z", "8.8.8.8"])
     assert_equal(options_target, options)
     assert_equal(["8.8.8.8"], items)
     
     options_target[:format] = "graphml"
-    options, items = PassiveDNS::CLI.parse_command_line(["-dptv", "-m", "8.8.8.8"])
+    options, items = PassiveDNS::CLI.parse_command_line(["-dpv", "-m", "8.8.8.8"])
     assert_equal(options_target, options)
     assert_equal(["8.8.8.8"], items)
     
@@ -167,7 +153,7 @@ Getting Help
   
   def test_help_debug_parsing
     options_target = {
-      :pdnsdbs => ["passivetotal", "tcpiputils", "virustotal"],
+      :pdnsdbs => ["passivetotal", "virustotal"],
       :format => "text",
       :sep => "\t",
       :recursedepth => 1,
@@ -180,19 +166,19 @@ Getting Help
       :configfile => "#{ENV['HOME']}/.passivedns-client"
     }
 
-    options, items = PassiveDNS::CLI.parse_command_line(["-dptv", "-h", "8.8.8.8"])
+    options, items = PassiveDNS::CLI.parse_command_line(["-dpv", "-h", "8.8.8.8"])
     assert_equal(options_target, options)
     assert_equal(["8.8.8.8"], items)
     
     options_target[:debug] = true
-    options, items = PassiveDNS::CLI.parse_command_line(["-dptv", "-h", "-v", "8.8.8.8"])
+    options, items = PassiveDNS::CLI.parse_command_line(["-dpv", "-h", "-v", "8.8.8.8"])
     assert_equal(options_target, options)
     assert_equal(["8.8.8.8"], items)
   end
   
   def test_state_recursion_parsing
     options_target = {
-      :pdnsdbs => ["passivetotal", "tcpiputils", "virustotal"],
+      :pdnsdbs => ["passivetotal", "virustotal"],
       :format => "text",
       :sep => "\t",
       :recursedepth => 5,
@@ -205,14 +191,14 @@ Getting Help
       :configfile => "#{ENV['HOME']}/.passivedns-client"
     }
 
-    options, items = PassiveDNS::CLI.parse_command_line(["-dptv", "-f", "test.db", "-r", "5", "-w", "30", "-l", "10", "8.8.8.8"])
+    options, items = PassiveDNS::CLI.parse_command_line(["-dpv", "-f", "test.db", "-r", "5", "-w", "30", "-l", "10", "8.8.8.8"])
     assert_equal(options_target, options)
     assert_equal(["8.8.8.8"], items)
   end
   
   def test_configuration_file
     options_target = {
-      :pdnsdbs => ["bfk"],
+      :pdnsdbs => ["virustotal"],
       :format => "text",
       :sep => "\t",
       :recursedepth => 1,
@@ -225,7 +211,6 @@ Getting Help
       :configfile => "#{ENV['HOME']}/.passivedns-client"
     }
     
-    
     options, items = PassiveDNS::CLI.parse_command_line(["--config", "#{ENV['HOME']}/.passivedns-client"])
     assert_equal(options_target, options)
     assert_equal([], items)

data/test/test_passivedns-client.rb

@@ -46,31 +46,6 @@ class TestPassiveDnsQuery < Minitest::Test
     end
   end
   
-  def test_BFK
-    assert_raises RuntimeError do
-      PassiveDNS::Client.new(['bfk'])
-    end
-    
-    assert_raises RuntimeError do
-      PassiveDNS::Provider::BFK.new(@cp['bfk'] || {})
-    end
-    
-    # refute_nil(d)
-    # rows = d.lookup("example.org",3)
-    # refute_nil(rows)
-    # refute_nil(rows.to_s)
-    # refute_nil(rows.to_xml)
-    # refute_nil(rows.to_json)
-    # refute_nil(rows.to_yaml)
-    # assert_equal(3, rows.length)
-    # rows = d.lookup("8.8.8.8")
-    # refute_nil(rows)
-    # refute_nil(rows.to_s)
-    # refute_nil(rows.to_xml)
-    # refute_nil(rows.to_json)
-    # refute_nil(rows.to_yaml)
-  end
-  
   def test_DNSDB
     PassiveDNS::Client.new(['dnsdb'])
     d = PassiveDNS::Provider::DNSDB.new(@cp['dnsdb'] || {})
@@ -109,81 +84,6 @@ class TestPassiveDnsQuery < Minitest::Test
     refute_nil(rows.to_yaml)
   end
   
-  def test_TCPIPUtils
-    PassiveDNS::Client.new(['tcpiputils'])
-    d = PassiveDNS::Provider::TCPIPUtils.new(@cp['tcpiputils'] || {})
-    refute_nil(d)
-    rows = d.lookup("example.org")
-    refute_nil(rows)
-    refute_nil(rows.to_s)
-    refute_nil(rows.to_xml)
-    refute_nil(rows.to_json)
-    refute_nil(rows.to_yaml)
-    rows = d.lookup("example.org",3)
-    refute_nil(rows)
-    refute_nil(rows.to_s)
-    refute_nil(rows.to_xml)
-    refute_nil(rows.to_json)
-    refute_nil(rows.to_yaml)
-    assert_equal(3, rows.length)
-    rows = d.lookup("8.8.8.8")
-    refute_nil(rows)
-    refute_nil(rows.to_s)
-    refute_nil(rows.to_xml)
-    refute_nil(rows.to_json)
-    refute_nil(rows.to_yaml)
-  end
-
-  def test_cn360
-    PassiveDNS::Client.new(['cn360'])
-    d = PassiveDNS::Provider::CN360.new(@cp['cn360'] || {})
-    refute_nil(d)
-    rows = d.lookup("example.org")
-    refute_nil(rows)
-    refute_nil(rows.to_s)
-    refute_nil(rows.to_xml)
-    refute_nil(rows.to_json)
-    refute_nil(rows.to_yaml)
-    rows = d.lookup("example.org",3)
-    refute_nil(rows)
-    refute_nil(rows.to_s)
-    refute_nil(rows.to_xml)
-    refute_nil(rows.to_json)
-    refute_nil(rows.to_yaml)
-    assert_equal(3, rows.length)
-    rows = d.lookup("8.8.8.8")
-    refute_nil(rows)
-    refute_nil(rows.to_s)
-    refute_nil(rows.to_xml)
-    refute_nil(rows.to_json)
-    refute_nil(rows.to_yaml)
-  end
-  
-  def test_nmemonic
-    PassiveDNS::Client.new(['mnemonic'])
-    d = PassiveDNS::Provider::Mnemonic.new(@cp['mnemonic'] || {})
-    refute_nil(d)
-    rows = d.lookup("example.org")
-    refute_nil(rows)
-    refute_nil(rows.to_s)
-    refute_nil(rows.to_xml)
-    refute_nil(rows.to_json)
-    refute_nil(rows.to_yaml)
-    rows = d.lookup("example.org",3)
-    refute_nil(rows)
-    refute_nil(rows.to_s)
-    refute_nil(rows.to_xml)
-    refute_nil(rows.to_json)
-    refute_nil(rows.to_yaml)
-    assert_equal(3, rows.length)
-    rows = d.lookup("8.8.8.8")
-    refute_nil(rows)
-    refute_nil(rows.to_s)
-    refute_nil(rows.to_xml)
-    refute_nil(rows.to_json)
-    refute_nil(rows.to_yaml)
-  end
-  
   def test_passivetotal
     PassiveDNS::Client.new(['passivetotal'])
     d = PassiveDNS::Provider::PassiveTotal.new(@cp['passivetotal'] || {})

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: passivedns-client
 version: !ruby/object:Gem::Version
-  version: 2.1.12
+  version: 2.1.13
 platform: ruby
 authors:
 - chrislee35
@@ -127,14 +127,10 @@ files:
 - lib/passivedns/client.rb
 - lib/passivedns/client/cli.rb
 - lib/passivedns/client/passivedb.rb
-- lib/passivedns/client/provider/bfk.rb
 - lib/passivedns/client/provider/circl.rb
-- lib/passivedns/client/provider/cn360.rb
 - lib/passivedns/client/provider/dnsdb.rb
-- lib/passivedns/client/provider/mnemonic.rb
 - lib/passivedns/client/provider/passivetotal.rb
 - lib/passivedns/client/provider/riskiq.rb
-- lib/passivedns/client/provider/tcpiputils.rb
 - lib/passivedns/client/provider/virustotal.rb
 - lib/passivedns/client/state.rb
 - lib/passivedns/client/version.rb

data/lib/passivedns/client/provider/bfk.rb

@@ -1,109 +0,0 @@
-require 'open-uri'
-
-module PassiveDNS #:nodoc: don't document this
-  # The Provider module contains all the Passive DNS provider client code
-  module Provider
-
-    # Queries BFK.de's passive DNS database
-    class BFK < PassiveDB
-      # Sets the modules self-reported name to "BFK.de"
-      def self.name
-        "BFK.de"
-      end
-      # Sets the configuration section name to "bfk"
-      def self.config_section_name
-        "bfk"
-      end
-      # Sets the command line database argument to "b"
-      def self.option_letter
-        "b"
-      end
-    
-      # :debug enables verbose logging to standard output
-      attr_accessor :debug
-      # === Options
-      # * :debug     Sets the debug flag for the module
-      # * "URL"      Alternate url for testing.  Defaults to "http://www.bfk.de/bfk_dnslogger.html?query="
-      #
-      # === Example Instantiation
-      #
-      #   options = {
-      #     :debug => true,
-      #     "URL" => "http://www.bfk.de/bfk_dnslogger.html?query="
-      #   }
-      #
-      #   PassiveDNS::Provider::BFK.new(options)
-      #
-      def initialize(options={})
-        @debug = options[:debug] || false
-        @timeout = options[:timeout] || 20
-        @base = options["URL"] || "http://www.bfk.de/bfk_dnslogger.html?query="
-        raise "Due to the EU GDPR policy, this service has been shut down until further notice."
-      end
-    
-      # Takes a label (either a domain or an IP address) and returns
-      # an array of PassiveDNS::PDNSResult instances with the answers to the query
-      def lookup(label, limit=nil)  
-        $stderr.puts "DEBUG: #{self.class.name}.lookup(#{label})" if @debug
-        Timeout::timeout(@timeout) {
-          t1 = Time.now
-          open(
-            @base+label,
-            "User-Agent" => "Ruby/#{RUBY_VERSION} passivedns-client rubygem v#{PassiveDNS::Client::VERSION}"
-          ) do |f|
-            t2 = Time.now
-            recs = parse(f.read,t2-t1)
-            if limit
-              recs[0,limit]
-            else
-              recs
-            end
-          end
-        }
-      rescue Timeout::Error
-        $stderr.puts "#{self.class.name} lookup timed out: #{label}"      
-      end
-    
-      private
-    
-      # parses the webpage returned by BFK to generate an array of PDNSResult
-      def parse(page,response_time)
-        line = page.unpack('C*').pack('U*').split(/<table/).grep(/ id=\"logger\"/)
-        return [] unless line.length > 0
-        line = line[0].gsub(/[\t\n]/,'').gsub(/<\/table.*/,'')
-        rows = line.split(/<tr.*?>/)
-        res = []
-        rows.collect do |row|
-          r = row.split(/<td>/).map{|x| x.gsub(/<.*?>/,'').gsub(/\&.*?;/,'')}[1,1000]
-          if r and r[0] =~ /\w/
-            # BFK includes the MX weight in the answer response. First, find the MX records, 
-            #    then dump the weight to present a consistent record name to the collecting 
-            #    array. Otherwise the other repositories will present the same answer and 
-            #    your results will become cluttered with duplicates.
-            if r[1] == "MX" then
-              # MX lines look like "5 mx.domain.tld", so split on the space and assign r[2] (:answer) to the latter part.
-              #s = r[2].split(/\w/).map{|x| x}[1,1000]
-              #  r[2] = s[1]
-              r[2] =~ /[0-9]+?\s(.+)/
-              s=$1
-              #puts "DEBUG: == BFK: MX Parsing Strip: Answer: " + r[2] + " : mod: " + s if @debug
-              r[2] = s
-            
-                ######### FIX BLANKS FOR MX
-            
-            end
-            query = r[0]
-            answer = r[2]
-            rrtype = r[1]
-            res << PDNSResult.new(self.class.name,response_time,query,answer,rrtype,'white')
-          end
-        end
-        res
-      rescue Exception => e
-        $stderr.puts "#{self.class.name} Exception: #{e}"
-        raise e
-      end
-
-    end
-  end
-end

data/lib/passivedns/client/provider/cn360.rb

@@ -1,122 +0,0 @@
-require 'net/http'
-require 'net/https'
-require 'openssl'
-require 'json'
-require 'digest/md5'
-require 'pp'
-
-module PassiveDNS #:nodoc: don't document this
-  # The Provider module contains all the Passive DNS provider client code
-  module Provider
-    # Queries 360.cn's passive DNS database
-    class CN360 < PassiveDB
-      # Sets the modules self-reported name to "360.cn"
-      def self.name
-        "360.cn"
-      end
-      # Sets the configuration section name to "cn360"
-      def self.config_section_name
-        "cn360"
-      end
-      # Sets the command line database argument to "3"
-      def self.option_letter
-        "3"
-      end
-    
-      # :debug enables verbose logging to standard output
-      attr_accessor :debug
-      # === Options
-      # * :debug       Sets the debug flag for the module
-      # * "API"        REQUIRED: http://some.web.address.for.their.api
-      # * "API_ID"     REQUIRED: a username that is given when you register
-      # * "API_KEY"    REQUIRED: a long and random password of sorts that is used along with the page request to generate a per page API key
-      #
-      # === Example Instantiation
-      #
-      #   options = {
-      #     :debug => true,
-      #     "API" => "http://some.web.address.for.their.api",
-      #     "API_ID" => "360user",
-      #     "API_KEY" => "360apikey"
-      #   }
-      #
-      #   PassiveDNS::Provider::CN360.new(options)
-      #
-      def initialize(options={})
-        @debug = options[:debug] || false
-        @timeout = options[:timeout] || 20
-        ["API", "API_ID", "API_KEY"].each do |opt|
-          if not options[opt]
-            raise "Field #{opt} is required.  See README.md"
-          end
-        end
-        @cp = options
-      end
-    
-      # Takes a label (either a domain or an IP address) and returns
-      # an array of PassiveDNS::PDNSResult instances with the answers to the query
-      def lookup(label, limit=10000)
-        table = "rrset"
-        if label =~ /^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}$/ or label =~ /^[0-9a-fA-F]+:[0-9a-fA-F:]+[0-9a-fA-F]$/
-          table = "rdata"
-        end
-        limit ||= 10000
-        path = "/api/#{table}/keyword/#{label}/count/#{limit}/"
-        Timeout::timeout(@timeout) {
-          url = @cp["API"]+path
-          url = URI.parse url
-          http = Net::HTTP.new(url.host, url.port)
-          http.use_ssl = (url.scheme == 'https')
-          http.verify_mode = OpenSSL::SSL::VERIFY_NONE # I hate doing this
-          http.verify_depth = 5
-          request = Net::HTTP::Get.new(url.path)
-          request.add_field("User-Agent", "Ruby/#{RUBY_VERSION} passivedns-client rubygem v#{PassiveDNS::Client::VERSION}")
-          request.add_field('Accept', 'application/json')
-          request.add_field("X-BashTokid", @cp["API_ID"])
-          token = Digest::MD5.hexdigest(path+@cp["API_KEY"])
-          $stderr.puts "DEBUG: cn360 url = #{url} token = #{token}" if @debug
-          request.add_field("X-BashToken", token)
-          t1 = Time.now
-          response = http.request(request)
-          t2 = Time.now
-          recs = parse_json(response.body, label, t2-t1)
-          if limit
-            recs[0,limit]
-          else
-            recs
-          end
-        }
-      rescue Timeout::Error
-        $stderr.puts "#{self.class.name} lookup timed out: #{label}"
-        recs
-      end
-      
-      private
-      
-      # parses the response of 360.cn's JSON reply to generate an array of PDNSResult
-      def parse_json(page,query,response_time=0)
-        res = []
-        data = JSON.parse(page)
-        if data.class == Hash and data['err']
-          raise "#{self.class.name} Error: #{data['err']}"
-        end
-        data.each do |row|
-          time_first = (row["time_first"]) ? Time.at(row["time_first"]) : nil
-          time_last = (row["time_last"]) ? Time.at(row["time_last"]) : nil
-          count = row["count"] || 0
-          query = row["rrname"]
-          answers = row["rdata"].gsub(/;$/,'').split(/;/)
-          rrtype = row["rrtype"]
-          answers.each do |answer|
-            res << PDNSResult.new(self.class.name, response_time, query, answer, rrtype, nil, time_first, time_last, count, TLPSecurityControl.new('yellow'))
-          end
-        end
-        res
-      rescue Exception => e
-        $stderr.puts "#{self.class.name} Exception: #{e}"
-        puts e.backtrace
-        raise e
-      end
-    end
-  end
-end

data/lib/passivedns/client/provider/mnemonic.rb

@@ -1,112 +0,0 @@
-# DESCRIPTION: Module to query Mnemonic's passive DNS repository
-# CONTRIBUTOR: Drew Hunt (pinowudi@yahoo.com)
-require 'net/http'
-require 'net/https'
-require 'openssl'
-
-module PassiveDNS #:nodoc: don't document this
-  # The Provider module contains all the Passive DNS provider client code
-  module Provider
-    # Queries Mnemonic's passive DNS database
-    class Mnemonic < PassiveDB
-      # Sets the modules self-reported name to "Mnemonic"
-      def self.name
-        "Mnemonic"
-      end
-      # Sets the configuration section name to "mnemonic"
-      def self.config_section_name
-        "mnemonic"
-      end
-      # Sets the command line database argument to "m"
-      def self.option_letter
-        "m"
-      end
-    
-      # :debug enables verbose logging to standard output
-      attr_accessor :debug
-      # === Options
-      # * :debug       Sets the debug flag for the module
-      # * "APIKEY"     The API key associated with Mnemonic for doing automated queries
-      # * "URL"        Alternate url for testing.  Defaults to "https://api.mnemonic.no/pdns/v3/"
-      #
-      # === Example Instantiation
-      #
-      #   options = {
-      #     :debug => true,
-      #     "APIKEY" => "01234567890abcdef01234567890abcdef012345",
-      #     "URL" => "https://api.mnemonic.no/pdns/v3/"
-      #   }
-      #
-      #   PassiveDNS::Provider::Mnemonic.new(options)
-      #
-      def initialize(options={})
-        @debug = options[:debug] || false
-        @timeout = options[:timeout] || 20
-        @apikey = options["APIKEY"]
-        @url = options["URL"] || "https://api.mnemonic.no/pdns/v3/"
-        if @url == "https://passivedns.mnemonic.no/api1/?apikey="
-          @url = "https://api.mnemonic.no/pdns/v3/"
-        end
-      end
-
-      # Takes a label (either a domain or an IP address) and returns
-      # an array of PassiveDNS::PDNSResult instances with the answers to the query
-      def lookup(label, limit=nil)
-        $stderr.puts "DEBUG: #{self.class.name}.lookup(#{label})" if @debug
-        Timeout::timeout(@timeout) {
-          url = "#{@url}#{label}"
-          $stderr.puts "DEBUG: #{self.class.name} url = #{url}" if @debug
-          url = URI.parse url
-          http = Net::HTTP.new(url.host, url.port)
-          http.use_ssl = (url.scheme == 'https')
-          http.verify_mode = OpenSSL::SSL::VERIFY_NONE
-          http.verify_depth = 5
-          request = Net::HTTP::Get.new(url.path)
-          request.add_field("User-Agent", "Ruby/#{RUBY_VERSION} passivedns-client rubygem v#{PassiveDNS::Client::VERSION}")
-          if @apikey
-            request.add_field("Argus-API-Key", @apikey)
-          end
-          t1 = Time.now
-          response = http.request(request)
-          t2 = Time.now
-          recs = parse_json(response.body, label, t2-t1)
-          if limit
-            recs[0,limit]
-          else
-            recs
-          end
-        }
-      rescue Timeout::Error
-        $stderr.puts "#{self.class.name} lookup timed out: #{label}"
-      end
-    
-      private
-    
-      # parses the response of mnemonic's JSON reply to generate an array of PDNSResult
-      def parse_json(page,query,response_time=0)
-        res = []
-        data = JSON.parse(page)
-        if data['data']
-          data['data'].each do |row|
-            if row['query']
-              query = row['query']
-              answer = row['answer']
-              rrtype = row['rrtype'].upcase
-              ttl = row['maxTtl'].to_i
-              firstseen = Time.at(row['firstSeenTimestamp'].to_i / 1000)
-              lastseen = Time.at(row['lastSeenTimestamp'].to_i / 1000)
-              tlp = row['tlp']
-              r = PDNSResult.new(self.class.name,response_time, query, answer, rrtype, ttl, firstseen, lastseen, tlp)
-              res << r
-            end
-          end
-        end
-        res
-      rescue Exception => e
-        $stderr.puts "#{self.class.name} Exception: #{e}"
-        raise e
-      end
-
-    end
-  end
-end

data/lib/passivedns/client/provider/tcpiputils.rb

@@ -1,129 +0,0 @@
-require 'net/http'
-require 'net/https'
-require 'openssl'
-require 'json'
-
-# Please read http://www.tcpiputils.com/terms-of-service under automated requests
-
-module PassiveDNS #:nodoc: don't document this
-  # The Provider module contains all the Passive DNS provider client code
-  module Provider
-    # Queries TCPIPUtils's passive DNS database
-    class TCPIPUtils < PassiveDB
-      # Sets the modules self-reported name to "TCPIPUtils"
-      def self.name
-        "TCPIPUtils"
-      end
-      # Sets the configuration section name to "tcpiputils"
-      def self.config_section_name
-        "tcpiputils"
-      end
-      # Sets the command line database argument to "t"
-      def self.option_letter
-        "t"
-      end
-
-      # :debug enables verbose logging to standard output
-      attr_accessor :debug
-      # === Options
-      # * :debug       Sets the debug flag for the module
-      # * "APIKEY"     REQUIRED: The API key associated with TCPIPUtils
-      # * "URL"      Alternate url for testing.  Defaults to  "https://www.utlsapi.com/api.php?version=1.0&apikey="
-      #
-      # === Example Instantiation
-      #
-      #   options = {
-      #     :debug => true,
-      #     "APIKEY" => "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef",
-      #     "URL" =>  "https://www.utlsapi.com/api.php?version=1.0&apikey="
-      #   }
-      #
-      #   PassiveDNS::Provider::TCPIPUtils.new(options)
-      #
-      def initialize(options={})
-        @debug = options[:debug] || false
-        @timeout = options[:timeout] || 20
-        @apikey = options["APIKEY"] || raise("#{self.class.name} requires an APIKEY.  See README.md")
-        @url = options["URL"] || "https://www.utlsapi.com/api.php?version=1.0&apikey="
-      end
-    
-      # Takes a label (either a domain or an IP address) and returns
-      # an array of PassiveDNS::PDNSResult instances with the answers to the query
-      def lookup(label, limit=nil)
-        $stderr.puts "DEBUG: #{self.class.name}.lookup(#{label})" if @debug
-        type = (label.match(/^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}$/)) ? "domainneighbors" : "domainipdnshistory"
-        url = "#{@url}#{@apikey}&type=#{type}&q=#{label}"
-        recs = []
-        Timeout::timeout(@timeout) {
-          url = URI.parse url
-          http = Net::HTTP.new(url.host, url.port)
-          http.use_ssl = (url.scheme == 'https')
-          http.verify_mode = OpenSSL::SSL::VERIFY_NONE
-          http.verify_depth = 5
-          request = Net::HTTP::Get.new(url.path+"?"+url.query)
-          request.add_field("User-Agent", "Ruby/#{RUBY_VERSION} passivedns-client rubygem v#{PassiveDNS::Client::VERSION}")
-          t1 = Time.now
-          response = http.request(request)
-          delta = (Time.now - t1).to_f
-          reply = JSON.parse(response.body)
-          if reply["status"] and reply["status"] == "succeed"
-            question = reply["data"]["question"]
-            recs = format_recs(reply["data"], question, delta)
-          elsif reply["status"] and reply["status"] == "error"
-            raise "#{self.class.name}: error from web API: #{reply["data"]}"
-          end
-          if limit
-            recs[0,limit]
-          else
-            recs
-          end
-        }
-      rescue Timeout::Error
-        $stderr.puts "#{self.class.name} lookup timed out: #{label}"
-      end
-    
-      private
-    
-      # translates the data structure derived from of tcpiputils's JSON reply
-      def format_recs(reply_data, question, delta)
-        recs = []
-        fieldname = nil
-        rrtype = nil
-        add_records = false
-        reply_data.each do |key, data|
-          case key
-          when "ipv4"
-            fieldname = "ip"
-            rrtype = "A"
-            add_records = true
-          when "ipv6"
-            fieldname = "ip"
-            rrtype = "AAAA"
-            add_records = true
-          when "dns"
-            fieldname = "dns"
-            rrtype = "NS"
-            add_records = true
-          when "mx"
-            fieldname = "dns"
-            rrtype = "MX"
-            add_records = true
-          when "domains"
-            data.each do |rec|
-              lastseen = (rec["updatedate"]) ? Date.parse(rec["updatedate"]) : nil
-              recs << PDNSResult.new(self.class.name, delta, rec, question, "A", nil, nil, lastseen, nil, 'yellow')
-            end
-          end
-          if add_records
-            data.each do |rec|
-              lastseen = (rec["updatedate"]) ? Date.parse(rec["updatedate"]) : nil
-              recs << PDNSResult.new(self.class.name, delta, question, rec[fieldname], rrtype, nil, nil, lastseen, nil, 'yellow')
-            end
-          end
-        end
-        recs
-      end
-
-    end
-  end
-end