passivedns-client 2.1.6 → 2.1.12

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: c9428d4dc084c7a7f0e734e9902791b469e6bd49
-  data.tar.gz: ddd6535a823edbf3e3a89bdd388782405fc6b52a
+SHA256:
+  metadata.gz: d8ea3feed82ed9f049ef8d7cd6ac78c087b94a630449cdbb3560fc28364ffe41
+  data.tar.gz: 37056ad39c2d721a3b5e6ee9f0347c5f1f79686f75eb15da99e2f155fc7b22d0
 SHA512:
-  metadata.gz: 6b0ca82916628a13baad54ce8c5ba263f33f8a661b43d659442656b5fe822914341b601b0203ecd8815898b096a4cb1d3c111c40a23032ec2ab7cc2d8b76713c
-  data.tar.gz: dcf33265380dedd7ad11f7d185036b7336dea02c7a89510d2d0cae4fac7ca3fafb945b965c616c658b1775c19a9252abe3a963e83feec2faff0b8b020c032900
+  metadata.gz: 61ca76c52991e1a9e131d51412f0a855301b9555cbd0d1d6e5fe9c0071c1bc2d0b1539926148c090ad31c7db1aaff544565537c4db2f59bfd449e6747786d655
+  data.tar.gz: 4bae5d3265ca1723c9170dc3e1ec726a6d855ff2324b4744692f0a9a486444751723fa907ce50ef9960900fa4236b53fdfb1484f04fc4a91c7dc1e6d61889756

data/README.md

@@ -115,6 +115,9 @@ Or use the included tool...
 	  -r# specifies the levels of recursion to pull. **WARNING** This is quite taxing on the pDNS servers, so use judiciously (never more than 3 or so) or find yourself blocked!
 	  -w# specifies the amount of time to wait, in seconds, between queries (Default: 0)
 	  -l <count> limits the number of records returned per passive dns database queried.
+    
+	Specifying a Configuration File
+	  --config <file> specifies a config file. default: #{ENV['HOME']}/.passivedns-client
 	  
 	Getting Help
 	  -v debugging information

data/lib/passivedns/client.rb

@@ -12,22 +12,56 @@ provider_path = File.dirname(__FILE__)+"/client/provider/*.rb"
 Dir.glob(provider_path).each do |provider|
   name = File.basename(provider, '.rb')
   require "passivedns/client/provider/#{name}.rb"
-  $passivedns_providers << name
+  if name != 'bfk'
+    $passivedns_providers << name
+  end
 end
 
 require 'configparser'
 
 module PassiveDNS # :nodoc:
+  
+  class SecurityControl
+    def allow(user_level)
+      raise "unimplemented"
+    end
+  end
+  
+  class TLPSecurityControl < SecurityControl
+    LEVELS = ['white','green','yellow','red']
+    
+    def initialize(tlp)
+      if tlp =~ /(white|green|yellow|red)/i
+        @tlp = tlp.downcase
+        @tlp_level = LEVELS.index(@tlp)
+      else
+        raise "Unknown TLP setting, #{tlp}"
+      end
+    end
+    
+    def allow(user_level)
+      user_level = LEVELS.index(user_level.downcase)
+      if user_level == nil
+        raise "Invalid user level, #{user_level}"
+      end
+      return(user_level >= @tlp_level)
+    end
+    
+    def to_s()
+      @tlp
+    end
+  end
+  
   # struct to contain the results from a PassiveDNS lookup
-	class PDNSResult < Struct.new(:source, :response_time, :query, :answer, :rrtype, :ttl, :firstseen, :lastseen, :count); end
+  class PDNSResult < Struct.new(:source, :response_time, :query, :answer, :rrtype, :ttl, :firstseen, :lastseen, :count, :security); end
 
   # coodinates the lookups accross all configured PassiveDNS providers
-	class Client
+  class Client
     
     # instantiate and configure all specified PassiveDNS providers
     # pdns        array of passivedns provider names, e.g., ["dnsdb","virustotal"]
     # configfile  filename of the passivedns-client configuration (this should probably be abstracted)
-		def initialize(pdns=$passivedns_providers, configfile="#{ENV['HOME']}/.passivedns-client")
+    def initialize(pdns=$passivedns_providers, configfile="#{ENV['HOME']}/.passivedns-client")
       cp = {}
       if File.exist?(configfile)
         cp = ConfigParser.new(configfile)
@@ -42,7 +76,7 @@ module PassiveDNS # :nodoc:
         end
       end
       
-			@pdnsdbs = []
+      @pdnsdbs = []
       pdns.uniq.each do |pd|
         if class_map[pd]
           @pdnsdbs << class_map[pd].new(cp[pd] || {})
@@ -51,38 +85,44 @@ module PassiveDNS # :nodoc:
         end
       end
 
-		end #initialize
-		
+    end #initialize
+    
     # set the debug flag
-		def debug=(d)
-			@pdnsdbs.each do |pdnsdb|
-				pdnsdb.debug = d
-			end
-		end
-		
+    def debug=(d)
+      @pdnsdbs.each do |pdnsdb|
+        pdnsdb.debug = d
+      end
+    end
+    
+    def timeout=(t)
+      @pdnsdbs.each do |pdnsdb|
+        pdnsdb.timeout = t
+      end
+    end
+    
     # perform the query lookup accross all configured PassiveDNS providers
-		def query(item, limit=nil)
-			threads = []
-			@pdnsdbs.each do |pdnsdb|
-				threads << Thread.new(item) do |q|
-					pdnsdb.lookup(q, limit)
-				end
-			end
-				
-			results = []
-			threads.each do |thr|
-				rv = thr.join.value
-				if rv
-					rv.each do |r|
-						if ["A","AAAA","NS","CNAME","PTR"].index(r.rrtype)
-							results << r
-						end
-					end
-				end
-			end
-			
-			return results
-		end #query
-		
-	end # Client
+    def query(item, limit=nil)
+      threads = []
+      @pdnsdbs.each do |pdnsdb|
+        threads << Thread.new(item) do |q|
+          pdnsdb.lookup(q, limit)
+        end
+      end
+        
+      results = []
+      threads.each do |thr|
+        rv = thr.join.value
+        if rv
+          rv.each do |r|
+            if ["A","AAAA","NS","CNAME","PTR"].index(r.rrtype)
+              results << r
+            end
+          end
+        end
+      end
+      
+      return results
+    end #query
+    
+  end # Client
 end # PassiveDNS

data/lib/passivedns/client/cli.rb

@@ -7,7 +7,7 @@ require 'pp'
 module PassiveDNS # :nodoc:
   # Handles all the command-line parsing, state tracking, and dispatching queries to the PassiveDNS::Client instance
   # CLInterface is aliased by CLI
-	class CLInterface
+  class CLInterface
     #  generates a mapping between the option letter for each PassiveDNS provider and the class
     def self.get_letter_map
       letter_map = {}
@@ -41,24 +41,25 @@ module PassiveDNS # :nodoc:
       origARGV = ARGV.dup
       ARGV.replace(args)
       opts = GetoptLong.new(
-      	[ '--help', '-h', GetoptLong::NO_ARGUMENT ],
-      	[ '--debug', '-v', GetoptLong::NO_ARGUMENT ],
-      	[ '--database', '-d', GetoptLong::REQUIRED_ARGUMENT ],
-	
-      	[ '--gdf', '-g', GetoptLong::NO_ARGUMENT ],
-      	[ '--graphviz', '-z', GetoptLong::NO_ARGUMENT ],
-      	[ '--graphml', '-m', GetoptLong::NO_ARGUMENT ],
-      	[ '--csv', '-c', GetoptLong::NO_ARGUMENT ],
-      	[ '--xml', '-x', GetoptLong::NO_ARGUMENT ],
-      	[ '--yaml', '-y', GetoptLong::NO_ARGUMENT ],
-      	[ '--json', '-j', GetoptLong::NO_ARGUMENT ],
-      	[ '--text', '-t', GetoptLong::NO_ARGUMENT ],
-      	[ '--sep', '-s', GetoptLong::REQUIRED_ARGUMENT ],
-	
-      	[ '--sqlite3', '-f', GetoptLong::REQUIRED_ARGUMENT ],
-      	[ '--recurse', '-r', GetoptLong::REQUIRED_ARGUMENT ],
-      	[ '--wait', '-w', GetoptLong::REQUIRED_ARGUMENT ],
-      	[ '--limit', '-l', GetoptLong::REQUIRED_ARGUMENT ]
+        [ '--help', '-h', GetoptLong::NO_ARGUMENT ],
+        [ '--debug', '-v', GetoptLong::NO_ARGUMENT ],
+        [ '--database', '-d', GetoptLong::REQUIRED_ARGUMENT ],
+  
+        [ '--gdf', '-g', GetoptLong::NO_ARGUMENT ],
+        [ '--graphviz', '-z', GetoptLong::NO_ARGUMENT ],
+        [ '--graphml', '-m', GetoptLong::NO_ARGUMENT ],
+        [ '--csv', '-c', GetoptLong::NO_ARGUMENT ],
+        [ '--xml', '-x', GetoptLong::NO_ARGUMENT ],
+        [ '--yaml', '-y', GetoptLong::NO_ARGUMENT ],
+        [ '--json', '-j', GetoptLong::NO_ARGUMENT ],
+        [ '--text', '-t', GetoptLong::NO_ARGUMENT ],
+        [ '--sep', '-s', GetoptLong::REQUIRED_ARGUMENT ],
+  
+        [ '--sqlite3', '-f', GetoptLong::REQUIRED_ARGUMENT ],
+        [ '--recurse', '-r', GetoptLong::REQUIRED_ARGUMENT ],
+        [ '--wait', '-w', GetoptLong::REQUIRED_ARGUMENT ],
+        [ '--limit', '-l', GetoptLong::REQUIRED_ARGUMENT ],
+        [ '--config', GetoptLong::REQUIRED_ARGUMENT ]
       )
 
       letter_map = get_letter_map
@@ -74,17 +75,18 @@ module PassiveDNS # :nodoc:
         :debug => false,
         :sqlitedb => nil,
         :limit => nil,
-        :help => false
+        :help => false,
+        :configfile => "#{ENV['HOME']}/.passivedns-client"
       }
 
       opts.each do |opt, arg|
-      	case opt
-      	when '--help'
-      		options[:help] = true
-      	when '--debug'
-      		options[:debug] = true
-      	when '--database'
-        	arg.split(//).each do |c|
+        case opt
+        when '--help'
+          options[:help] = true
+        when '--debug'
+          options[:debug] = true
+        when '--database'
+          arg.split(//).each do |c|
             if c == ','
               next
             elsif letter_map[c]
@@ -94,60 +96,62 @@ module PassiveDNS # :nodoc:
               usage(letter_map)
             end
           end
-      	when '--gdf'
-      		options[:format] = 'gdf'
-      	when '--graphviz'
-      		options[:format] = 'graphviz'
-      	when '--graphml'
-      		options[:format] = 'graphml'
-      	when '--csv'
-      		options[:format] = 'text'
-      		options[:sep] = ','
-      	when '--yaml'
-      		options[:format] = 'yaml'
-      	when '--xml'
-      		options[:format] = 'xml'
-      	when '--json'
-      		options[:format] = 'json'
-      	when '--text'
-      		options[:format] = 'text'
-      	when '--sep'
-      		options[:sep] = arg
-      	when '--recurse'
-      		options[:recursedepth] = arg.to_i
-      	when '--wait'
-      		options[:wait] = arg.to_i
-      	when '--sqlite3'
-      		options[:sqlitedb] = arg
-      	when '--limit'
-      		options[:limit] = arg.to_i
-      	else
-      		options[:help] = true
-      	end
+        when '--gdf'
+          options[:format] = 'gdf'
+        when '--graphviz'
+          options[:format] = 'graphviz'
+        when '--graphml'
+          options[:format] = 'graphml'
+        when '--csv'
+          options[:format] = 'text'
+          options[:sep] = ','
+        when '--yaml'
+          options[:format] = 'yaml'
+        when '--xml'
+          options[:format] = 'xml'
+        when '--json'
+          options[:format] = 'json'
+        when '--text'
+          options[:format] = 'text'
+        when '--sep'
+          options[:sep] = arg
+        when '--recurse'
+          options[:recursedepth] = arg.to_i
+        when '--wait'
+          options[:wait] = arg.to_i
+        when '--sqlite3'
+          options[:sqlitedb] = arg
+        when '--limit'
+          options[:limit] = arg.to_i
+        when '--config'
+          options[:configfile] = arg
+        else
+          options[:help] = true
+        end
       end
       args = ARGV.dup
       ARGV.replace(origARGV)
 
       if options[:pdnsdbs].length == 0
-      	options[:pdnsdbs] << "bfk"
+        options[:pdnsdbs] << "bfk"
       end
 
       if options[:pdnsdbs].index("bfk") and options[:recursedepth] > 1 and options[:wait] < 60
-      	options[:wait] = 60
-      	$stderr.puts "Enforcing a minimal 60 second wait when using BFK for recursive crawling"
+        options[:wait] = 60
+        $stderr.puts "Enforcing a minimal 60 second wait when using BFK for recursive crawling"
       end
 
       if options[:debug]
-      	$stderr.puts "Using the following databases: #{options[:pdnsdbs].join(", ")}"
-      	$stderr.puts "Recursions: #{options[:recursedepth]}, Wait time: #{options[:wait]}, Limit: #{options[:limit] or 'none'}"
-      	if options[:format] == "text" or options[:format] == "csv"
-          	$stderr.puts "Output format: #{options[:format]} (sep=\"#{options[:sep]}\")"
-      	else
-          	$stderr.puts "Output format: #{options[:format]}"
-      	end
-      	if ENV['http_proxy']
-      		$stderr.puts "Using proxy settings: http_proxy=#{ENV['http_proxy']}, https_proxy=#{ENV['https_proxy']}"
-      	end
+        $stderr.puts "Using the following databases: #{options[:pdnsdbs].join(", ")}"
+        $stderr.puts "Recursions: #{options[:recursedepth]}, Wait time: #{options[:wait]}, Limit: #{options[:limit] or 'none'}"
+        if options[:format] == "text" or options[:format] == "csv"
+            $stderr.puts "Output format: #{options[:format]} (sep=\"#{options[:sep]}\")"
+        else
+            $stderr.puts "Output format: #{options[:format]}"
+        end
+        if ENV['http_proxy']
+          $stderr.puts "Using proxy settings: http_proxy=#{ENV['http_proxy']}, https_proxy=#{ENV['https_proxy']}"
+        end
       end
       
       [options, args]
@@ -158,36 +162,39 @@ module PassiveDNS # :nodoc:
     def self.usage(letter_map)
       databases = letter_map.keys.sort.join("")
       help_text = ""
-    	help_text << "Usage: #{$0} [-d [#{databases}]] [-g|-v|-m|-c|-x|-y|-j|-t] [-os <sep>] [-f <file>] [-r#|-w#|-v] [-l <count>] <ip|domain|cidr>\n"
+      help_text << "Usage: #{$0} [-d [#{databases}]] [-g|-v|-m|-c|-x|-y|-j|-t] [-os <sep>] [-f <file>] [-r#|-w#|-v] [-l <count>] [--config <file>] <ip|domain|cidr>\n"
       help_text << "Passive DNS Providers\n"
-    	help_text << "  -d#{databases} uses all of the available passive dns database\n"
+      help_text << "  -d#{databases} uses all of the available passive dns database\n"
       letter_map.keys.sort.each do |l|
         help_text << "  -d#{l} use #{letter_map[l][0]}\n"
       end
-    	help_text << "  -dvt uses VirusTotal and TCPIPUtils (for example)\n"
-    	help_text << "\n"
+      help_text << "  -dvt uses VirusTotal and TCPIPUtils (for example)\n"
+      help_text << "\n"
       help_text << "Output Formatting\n"
-    	help_text << "  -g link-nodal GDF visualization definition\n"
-    	help_text << "  -z link-nodal graphviz visualization definition\n"
-    	help_text << "  -m link-nodal graphml visualization definition\n"
-    	help_text << "  -c CSV\n"
-    	help_text << "  -x XML\n"
-    	help_text << "  -y YAML\n"
-    	help_text << "  -j JSON\n"
-    	help_text << "  -t ASCII text (default)\n"
-    	help_text << "  -s <sep> specifies a field separator for text output, default is tab\n"
-    	help_text << "\n"
+      help_text << "  -g link-nodal GDF visualization definition\n"
+      help_text << "  -z link-nodal graphviz visualization definition\n"
+      help_text << "  -m link-nodal graphml visualization definition\n"
+      help_text << "  -c CSV\n"
+      help_text << "  -x XML\n"
+      help_text << "  -y YAML\n"
+      help_text << "  -j JSON\n"
+      help_text << "  -t ASCII text (default)\n"
+      help_text << "  -s <sep> specifies a field separator for text output, default is tab\n"
+      help_text << "\n"
       help_text << "State and Recursion\n"
-    	help_text << "  -f[file] specifies a sqlite3 database used to read the current state - useful for large result sets and generating graphs of previous runs.\n"
-    	help_text << "  -r# specifies the levels of recursion to pull. **WARNING** This is quite taxing on the pDNS servers, so use judiciously (never more than 3 or so) or find yourself blocked!\n"
-    	help_text << "  -w# specifies the amount of time to wait, in seconds, between queries (Default: 0)\n"
-    	help_text << "  -l <count> limits the number of records returned per passive dns database queried.\n"
-    	help_text << "\n"
+      help_text << "  -f[file] specifies a sqlite3 database used to read the current state - useful for large result sets and generating graphs of previous runs.\n"
+      help_text << "  -r# specifies the levels of recursion to pull. **WARNING** This is quite taxing on the pDNS servers, so use judiciously (never more than 3 or so) or find yourself blocked!\n"
+      help_text << "  -w# specifies the amount of time to wait, in seconds, between queries (Default: 0)\n"
+      help_text << "  -l <count> limits the number of records returned per passive dns database queried.\n"
+      help_text << "\n"
+      help_text << "Specifying a Configuration File\n"
+      help_text << "  --config <file> specifies a config file. default: #{ENV['HOME']}/.passivedns-client\n"
+      help_text << "\n"
       help_text << "Getting Help\n"
       help_text << "  -h hello there.  This option produces this helpful help information on how to access help.\n"
-    	help_text << "  -v debugging information\n"
+      help_text << "  -v debugging information\n"
       
-    	help_text
+      help_text
     end
     
     # performs a stateful, recursive (if desired) passive DNS lookup against all specified providers
@@ -196,59 +203,60 @@ module PassiveDNS # :nodoc:
       wait = options[:wait]
       debug = options[:debug]
       limit = options[:limit]
-    	puts "pdnslookup: #{state.level} #{recursedepth}" if debug
-    	level = 0
-    	while level < recursedepth
-    		puts "pdnslookup: #{level} < #{recursedepth}" if debug
-    		state.each_query(recursedepth) do |q|
-    			rv = pdnsclient.query(q,limit)
-    			if rv
-    				rv.each do |r|
-    					if ["A","AAAA","NS","CNAME","PTR"].index(r.rrtype)
-    						puts "pdnslookup: #{r.to_s}" if debug
-    						state.add_result(r)
-    					end
-    				end
-    			else
-    				state.update_query(rv,'failed')
-    			end
-    			sleep wait if level < recursedepth
-    		end
-    		level += 1
-    	end
-    	state
+      puts "pdnslookup: #{state.level} #{recursedepth}" if debug
+      level = 0
+      while level < recursedepth
+        puts "pdnslookup: #{level} < #{recursedepth}" if debug
+        state.each_query(recursedepth) do |q|
+          rv = pdnsclient.query(q,limit)
+          if rv
+            rv.each do |r|
+              if ["A","AAAA","NS","CNAME","PTR"].index(r.rrtype)
+                puts "pdnslookup: #{r.to_s}" if debug
+                state.add_result(r)
+              end
+            end
+          else
+            state.update_query(rv,'failed')
+          end
+          sleep wait if level < recursedepth
+        end
+        level += 1
+      end
+      state
     end
     
     # returns a string transforming all the PassiveDNS::PDNSResult stored in the state object into text/xml/json/etc.
     def self.results_to_s(state,options)
       format = options[:format]
       sep = options[:sep]
-    	case format
-    	when 'text'
-    		PassiveDNS::PDNSResult.members.join(sep)+"\n"+state.to_s(sep)
-    	when 'yaml'
-    		state.to_yaml
-    	when 'xml'
-    		state.to_xml
-    	when 'json'
-    		state.to_json
-    	when 'gdf'
-    		state.to_gdf
-    	when 'graphviz'
-    		state.to_graphviz
-    	when 'graphml'
-    		state.to_graphml
-    	end
+      case format
+      when 'text'
+        PassiveDNS::PDNSResult.members.join(sep)+"\n"+state.to_s(sep)
+      when 'yaml'
+        state.to_yaml
+      when 'xml'
+        state.to_xml
+      when 'json'
+        state.to_json
+      when 'gdf'
+        state.to_gdf
+      when 'graphviz'
+        state.to_graphviz
+      when 'graphml'
+        state.to_graphml
+      end
     end
     
     # create a state instance
     def self.create_state(sqlitedb=nil)
       state = nil
       if sqlitedb
-      	state = PassiveDNS::PDNSToolStateDB.new(sqlitedb)
+        state = PassiveDNS::PDNSToolStateDB.new(sqlitedb)
       else
-      	state = PassiveDNS::PDNSToolState.new
+        state = PassiveDNS::PDNSToolState.new
       end
+      state
     end
     
     # main method, takes command-line arguments and performs the desired queries and outputs
@@ -257,24 +265,24 @@ module PassiveDNS # :nodoc:
       if options[:help]
         return usage(get_letter_map)
       end
-  		if options[:recursedepth] > 3
-  			$stderr.puts "WARNING:  a recursedepth of > 3 can be abusive, please reconsider: sleeping 60 seconds for sense to come to you (hint: hit CTRL-C)"
-  			sleep 60
-  		end
+      if options[:recursedepth] > 3
+        $stderr.puts "WARNING:  a recursedepth of > 3 can be abusive, please reconsider: sleeping 60 seconds for sense to come to you (hint: hit CTRL-C)"
+        sleep 60
+      end
       state = create_state(options[:sqlitedb])
       state.debug = options[:debug]
 
-      pdnsclient = PassiveDNS::Client.new(options[:pdnsdbs])
+      pdnsclient = PassiveDNS::Client.new(options[:pdnsdbs], options[:configfile])
       pdnsclient.debug = options[:debug]
       
       if items.length > 0
-      	items.each do |arg|
-      		state.add_query(arg,'pending',0)
-      	end
+        items.each do |arg|
+          state.add_query(arg,'pending',0)
+        end
       else
-      	$stdin.each_line do |l|
-      		state.add_query(l.chomp,'pending',0)
-      	end
+        $stdin.each_line do |l|
+          state.add_query(l.chomp,'pending',0)
+        end
       end
       pdnslookup(state,pdnsclient,options)
       results_to_s(state,options)