wmap 2.4.4

data/bin/wdel

@@ -0,0 +1,25 @@
+#!/usr/bin/env ruby
+# Executable to delete a new site into the tracking data repository
+# This is useful as a self-correction mechanism to flag out unique website in a constant way
+require "wmap"
+
+def print_usage
+	puts "Program to delete an entry from the local data repository. Usage: wdel [site]"
+end
+
+puts Wmap.banner
+print_usage
+Log_dir=File.dirname(__FILE__)+'/../logs/'
+Wmap.wlog("Execute the command: wdel #{ARGV[0]}","wdel",Log_dir+"wmap.log")
+
+st=Wmap::SiteTracker.instance
+abort "Incorrect program argument!" unless ARGV.length==1
+
+# Evaluate the argument and update the data store accordingly
+if st.is_site?(ARGV[0])
+	st.delete(ARGV[0])
+	st.save!
+	st=nil
+else
+	abort "Unknown argument format: #{ARGV[0]}"
+end

data/bin/wdump

@@ -0,0 +1,21 @@
+#!/usr/bin/env ruby
+# Wmap data repository Dump - Dump out a list of unique websites stored in the current tracking data repository.  
+# Usage: wdump <output file>
+require "wmap"
+
+def print_usage
+	puts "Program to dump out unique website assets from the local repository. Usage: wdump <output file>"
+end
+
+puts Wmap.banner
+Log_dir=File.dirname(__FILE__)+'/../logs/'
+Wmap.wlog("Execute the command: wdump #{ARGV[0]}","wdump",Log_dir+"wmap.log")
+
+unless ARGV.length==1
+	print_usage  
+	abort "Program argument error. Please check your input and try again. " 
+end
+# dump out the unique target list in .csv format
+continue=Wmap.dump(ARGV[0]+".csv")
+# continue to dump out the list in the .xml format if previous step is successful
+Wmap.dump_xml(ARGV[0]+".xml") if continue

data/bin/wmap

@@ -0,0 +1,151 @@
+#!/usr/bin/env ruby
+# Wmap main executable - intelligent enough to handle most command argument inputs from the user.
+# The discovery result is automatically compared and saved into the the tracking data repository.
+#
+# Usage: wmap <Target Host | URL | IP | CIDR | or a seed file with any of the above combo> <Optional Discovery Result Directory>
+require "wmap"
+
+def print_usage
+	abort "Program to perform website asset discovery and tracking. \nUsage: wmap <Target Host | URL | IP | CIDR | or a seed file with any of the above combo> <Optional Discovery Result Directory>"
+end
+
+# preparing - spit out the program banner
+puts Wmap.banner
+if ARGV.length == 1
+	# Log the command entry
+	Log_dir=File.dirname(__FILE__)+'/../logs/'
+elsif ARGV.length == 2
+	# Log to the instance running directory
+	Log_dir=ARGV[1] + '/logs/'
+	Dir.mkdir(Log_dir) unless Dir.exist?(Log_dir)
+end
+
+Wmap.wlog("Execute the command: wmap #{ARGV[0]}","wmap",Log_dir+"wmap.log")
+print_usage unless (ARGV.length==1 or ARGV.length==2)
+urls = Array.new
+# first step - construct the host list
+scanner = Wmap::PortScanner.new(:verbose=>false, :socket_timeout=>600) # default time-out of 600 milliseconds
+hosts=Array.new
+if File.exist?(ARGV[0])
+	puts "Parsing the discovery seed file: \"#{ARGV[0]}\" "
+	seeds=scanner.file_2_list(ARGV[0])-[nil,""]
+	domains=Array.new
+	cidrs=Array.new
+	raise "Error: empty seed file or no legal entry found!" if seeds.nil? or seeds.empty?
+	seeds.map do |x|
+		x=x.split(%r{(,|\s+)})[0]
+		urls.push(x) if scanner.is_url?(x)
+		domains.push(x) if scanner.is_domain_root?(x) or Wmap.sub_domain_known?(x)
+		# invoke bruter if the hostname contains a numeric number.
+		domains.push(x) if scanner.is_fqdn?(x) and (x.split('.')[0] =~ /\d+/)
+		hosts.push(x) if scanner.is_fqdn?(x) or scanner.is_ip?(x)
+		cidrs.push(x) if scanner.is_cidr?(x)
+	end
+	puts "Parsing done. "
+	hosts+=Wmap::DnsBruter.new(:verbose=>false).dns_brute_workers(domains.uniq).values.flatten if domains.size > 0
+	cidrs.map { |x| hosts+= scanner.cidr_2_ips(x) } if cidrs.size > 0
+elsif scanner.is_url?(ARGV[0])
+	puts "Processing the URL: #{ARGV[0]}"
+	urls.push(ARGV[0])
+elsif Wmap.domain_known?(ARGV[0]) or Wmap.sub_domain_known?(ARGV[0])
+	puts "Processing the domain: #{ARGV[0]}"
+	hosts+=Wmap::DnsBruter.new(:verbose=>false).dns_brute_worker(ARGV[0]).values.flatten
+elsif scanner.is_fqdn?(ARGV[0])
+	puts "Processing the host: #{ARGV[0]}"
+	hosts.push(ARGV[0])
+	my_hosts=Wmap::DnsBruter.new(:verbose=>false).dns_brute_worker(ARGV[0]).values.flatten if (ARGV[0].split('.')[0] =~ /\d+/)
+	hosts+=my_hosts unless my_hosts.nil?
+elsif scanner.is_cidr?(ARGV[0])
+	puts "Processing the network block: #{ARGV[0]}"
+	hosts+=scanner.cidr_2_ips(ARGV[0])
+elsif scanner.is_ip?(ARGV[0])
+	hosts.push(ARGV[0])
+else
+	print_usage
+end
+
+# second step - update the hosts repository
+if ARGV.length == 1
+	puts puts "Invoke the HostTracker."
+	host_tracker = Wmap::HostTracker.new(:verbose=>true)
+elsif ARGV.length == 2
+	puts "Invoke the HostTracker with optional directory setter."
+	host_tracker = Wmap::HostTracker.new(:verbose=>true, :data_dir => ARGV[1])
+else
+	aborts "Error firing up HostTracker instance!"
+end
+hosts.uniq!
+if hosts.size > 0
+	hostnames=hosts.dup.delete_if { |h| host_tracker.is_ip?(h) }
+	if hostnames.size > 0
+		puts "Update the local hosts data repository."
+		new_hosts=host_tracker.adds(hostnames)
+		host_tracker.save! if new_hosts.size>0
+	end
+end
+host_tracker=nil
+
+# third step - port discovery on the above host list, and to build the URL seeds
+puts "Build up URL list for the web crawler ..."
+urls0=scanner.scans(hosts)
+urls+=urls0
+urls.uniq!
+scanner=nil
+
+# fourth step - crawling on the URL seeds
+if ARGV.length == 1
+	puts "Fire up the crawler."
+	crawler = Wmap::UrlCrawler.new(:verbose=>false)
+elsif ARGV.length == 2
+	puts "Fire up the crawler with the optional directory setter."
+	crawler = Wmap::UrlCrawler.new(:data_dir => ARGV[1])
+else
+	aborts "Error firing up UrlCrawler instance!"
+end
+Wmap.wlog(urls, "wmap", Log_dir+"url_seeds.log") if urls.size > 0   # save port scan results for debugging
+crawler.crawls(urls) if urls.size>0
+dis_urls=crawler.discovered_urls_by_crawler
+#c_start=crawler.crawl_start
+#c_done=crawler.crawl_done
+dis_sites=Hash.new
+unless dis_urls.empty?
+	dis_urls.keys.map do |url|
+		site=crawler.url_2_site(url)
+		dis_sites[site]=true unless dis_sites.key?(site)
+	end
+end
+puts "Discovered sites: "
+
+if dis_sites.empty?
+	puts "No web site is discovered. "
+else
+	dis_sites.keys.map {|x| puts x}
+end
+
+# fifth step - trace the discovery results into a local log file for debugging and other purposes
+Wmap.wlog(dis_urls.keys, "wmap", Log_dir+"discovered_urls.log") unless dis_urls.empty?
+Wmap.wlog(dis_sites.keys, "wmap", Log_dir+"discovered_sites.log") unless dis_sites.empty?
+#crawler.wlog(c_start.keys,Log_dir+"crawler.log")
+#crawler.wlog(c_done.keys,Log_dir+"crawler.log")
+crawler=nil
+
+# sixth step - save discovery results into the inventory data repository
+case dis_sites.keys
+when nil,[]
+	puts "No new site found. There is no change to the site tracking data repository. "
+else
+	puts "Automatically save the discovery results into the site tracking data repository: "
+	if ARGV.length == 1
+		puts "Start the SiteTracker. "
+		inventory=Wmap::SiteTracker.new
+	elsif ARGV.length == 2
+		puts "Start the SiteTracker with the optional directory setter. "
+		inventory=Wmap::SiteTracker.new(:data_dir => ARGV[1])
+	else
+		aborts "Error firing up SiteTracker instance!"
+	end
+	new_sites=inventory.adds(dis_sites.keys)
+	inventory.save! if new_sites.size>0
+	inventory=nil
+	puts "Done! New found sites are successfully saved. " if new_sites.size > 0
+end

data/bin/wscan

@@ -0,0 +1,32 @@
+#!/usr/bin/env ruby
+# Executable to perform port scanning by using build-in tcp port scanner
+# This is useful to detect the web service
+require "wmap"
+
+def print_usage
+	puts "Program to port scanning remote hops(s) or IP(s). Usage: wscan [IP | File with list of IPs]"
+end
+
+puts Wmap.banner
+print_usage
+Log_dir=File.dirname(__FILE__)+'/../logs/'
+Wmap.wlog("Execute the command: wscan #{ARGV[0]}","wscan",Log_dir+"wmap.log")
+
+sites=Array.new
+ps=Wmap::PortScanner.new
+abort "Incorrect program argument!" unless ARGV.length==1 
+
+# Evaluate the argument and update the data psore accordingly
+if File.exist?(ARGV[0])
+	targets=ps.file_2_list(ARGV[0])
+	sites=ps.scans(targets)
+elsif ps.is_ip?(ARGV[0]) or ps.is_fqdn?(ARGV[0])
+	sites=ps.scan(ARGV[0])
+else
+	print_usage and abort "Unknown argument format: #{ARGV[0]}."
+end
+ps=nil
+if sites.count>0
+	puts "Done. Discovered sites:"
+	sites.map {|x| puts x} 
+end

data/data/cidrs

@@ -0,0 +1,2 @@
+# Local cidrs file created by Wmap::CidrTracker.save method at: 2015-07-07 12:23:56 -0400
+Network CIDR, CIDR RIPE Reference Text, CIDR NETNAME

data/data/deactivated_sites

@@ -0,0 +1 @@
+# Local site store created by class Wmap::SiteTracker::DeactivatedSite method save_sites_to_file! at: 2015-04-27 08:43:06 -0400

data/data/domains

@@ -0,0 +1,2 @@
+# Local domains file created by class Wmap::DomainTracker method save_domains_to_file! at: 2015-07-07 12:24:49 -0400
+# domain name, free zone transfer detected?

data/data/hosts

@@ -0,0 +1 @@
+# local hosts file created by the Wmap::HostTracker class save_known_hosts_to_file! method at: 2015-04-21 08:59:33 -0400

data/data/prime_hosts

@@ -0,0 +1 @@
+# local hosts file created by the Wmap::HostTracker::PrimaryHost class save_known_hosts_to_file! method at: 2015-07-07 12:25:34 -0400

data/data/sites

@@ -0,0 +1,2 @@
+# Local site store created by class Wmap::SiteTracker method save_sites_to_file! at: 2015-04-22 14:03:45 -0400
+# Website,Primary IP,Port,Hosting Status,Server,Response Code,Site MD5 Finger-print,Site Redirection,Timestamp

data/data/sub_domains

@@ -0,0 +1,2 @@
+# Local domains file created by class Wmap::DomainTracker::SubDomain method save_sub_domains_to_file! at: 2018-01-08 10:18:24 -0500
+# domain name, free zone transfer detected?

data/demos/bruter.rb

@@ -0,0 +1,27 @@
+# Brute-forcing multiple domains at the same time, the purpose is to extract a valid host list
+# Usage: ruby dns_brute.rb [file with list of domains]
+require "wmap"
+
+f_rpt=".rpt.txt"
+# Step 1 - obtain list of domains to be brute-forced on
+host_tracker=Wmap::HostTracker.new
+root_domains=host_tracker.dump_root_domains
+sub_domains=host_tracker.instance.dump_sub_domains
+# Step 2 - multi-thread brute forcer works on known domains and sub-domains
+k=Wmap::DnsBruter.new(:verbose=>true, :max_parallel=>50)
+#hosts=k.dns_brute_file(ARGV[0])
+results=k.dns_brute_workers(sub_domains+root_domains)
+k=nil
+#hosts=hosts1+hosts2
+# Step 3 - save results to a local file for debugging
+f=File.open(f_rpt,"w")
+results.each_pair do |key,value|
+  f.write("#{value}\n")
+end
+f.close
+puts "Brute force results are saved successfully: #{f_rpt}"
+
+# Step 4 - now update the local hosts table accordingly
+host_tracker.bulk_add(results.values.flatten)
+host_tracker.instance.save!
+host_tracker=nil

data/demos/dns_brutes.rb

@@ -0,0 +1,28 @@
+# Brute-forcing multiple domains at the same time, the purpose is to extract a valid host list
+# Usage: ruby dns_brute.rb [file with list of domains]
+require "wmap"
+
+f_rpt=".rpt.txt"
+# Step 1 - obtain list of domains to be brute-forced on
+tracker=Wmap::HostTracker.new
+root_domains=tracker.dump_root_domains
+sub_domains=tracker.dump_sub_domains
+# Step 2 - multi-thread brute forcer works on known domains and sub-domains 
+k=Wmap::DnsBruter.new(:verbose=>false, :max_parallel=>50)
+#hosts=k.dns_brute_file(ARGV[0])
+#hosts = k.dns_brute_domains(root_domains)
+hosts=k.dns_brute_domains(sub_domains)
+k=nil
+#hosts=hosts1+hosts2
+# Step 3 - save results to a local file for debugging
+f=File.open(f_rpt,"w")
+hosts.map do |x|
+  f.write("#{x}\n")
+end
+f.close
+puts "Brute force results are saved successfully: #{f_rpt}"
+
+# Step 4 - now update the local hosts table accordingly
+tracker.bulk_add(hosts)
+tracker.save!
+tracker=nil

data/demos/filter_cidr.rb

@@ -0,0 +1,18 @@
+# Sample CIDR Tracker - Given a trusted IP, print out all tracked CIDR information  
+# Usage: ruby filter_cidr.rb [file_web_hosts] 
+require	"wmap"
+
+puts "IP, CIDR, CIDR Netname, CIDR Reference"
+myDis = Wmap::CidrTracker.new(:verbose=>false)
+
+f_ips = File.open(ARGV[0], 'r:iso-8859-1')
+f_ips.each do |line|
+	ip=line.chomp.split(',')[1]
+	if myDis.is_ip?(ip) 
+		tracker=myDis.track(ip)
+		puts "#{line.chomp},#{tracker['cidr']},#{tracker['netname']},#{tracker['ref']}"
+	else
+		puts "#{line.chomp},,,"
+	end
+end
+f_ips.close

data/demos/filter_crawls.rb

@@ -0,0 +1,5 @@
+require "wmap"
+
+k=Wmap::UrlCrawler.new(:verbose=>true)
+sites=k.crawls(k.file_2_list(ARGV[0]))
+k.wlog(sites,".crawl_sites") 

data/demos/filter_domain.rb

@@ -0,0 +1,25 @@
+# Input file is a list of hosts or domains, output is a list of unknown hosts / domains
+# Usage: ruby filter_domain_x.rb [file_host]
+require	"wmap"
+
+puts Wmap.banner
+puts "Host,Domain,IP,Trusted CIDR,Trusted CIDR Netname"
+myD = Wmap::DomainTracker.new
+f_urls = File.open(ARGV[0], 'r')
+f_urls.each do |line|
+	url=line.chomp
+	host=myD.url_2_host(url)
+	next if host.nil?
+	domain=myD.domain_root(host)
+	next if domain.nil?
+	if myD.domain_known?(domain)
+		#puts url
+		next
+	else
+		#next
+		tracker=Wmap.track(host)
+		puts "#{host}, #{domain}, #{tracker['ip']}, #{tracker['cidr']}, #{tracker['netname']}"
+	end
+end
+
+f_urls.close

data/demos/filter_geoip.rb

@@ -0,0 +1,26 @@
+# Perform GEOIP info lookup, based on Maxmind GeoIP database
+# Usage: ruby filter_geoip.rb [file_wdump_csv] 
+require	"wmap"
+
+verbose=false
+tracker = Wmap::GeoIPTracker.new(:verbose=>false)
+
+#puts "IP, GeoIP Location"
+f_ips = File.open(ARGV[0], 'r')
+f_ips.each do |line|
+	ip=line.chomp.split(',')[1]
+	info=String.new
+	if tracker.is_ip?(ip)
+		puts "IP: #{ip}" if verbose
+		ctr=tracker.country(ip)
+		info = info + " " + ctr.country_code3 unless ctr.country_code3.nil?
+		citi=tracker.city(ip)
+		info=info+" "+citi.region_name unless citi.nil? or citi.region_name.nil?
+		info=info+" "+citi.city_name unless citi.nil? or citi.city_name.nil?
+		info=info+" "+citi.postal_code unless citi.nil? or citi.postal_code.nil?
+		puts "#{line.chomp}, #{info}"
+	else
+		puts "#{line.chomp},"
+	end
+end
+f_ips.close

data/demos/filter_known_services.rb

@@ -0,0 +1,59 @@
+# Compare the scan lists and flag out the new site
+# Usage: filter_known_@services.rb [last quarter list] [this quarter list]
+
+require "wmap"
+
+
+# Create a known service map by parsing last quarter scan list
+def parse_old
+	host_tracker=Wmap::HostTracker.new
+	@services=Hash.new
+	f_site=File.open(ARGV[0],'r')
+	f_site.each do |line|
+		site=line.chomp.strip
+		site=host_tracker.url_2_site(site)
+		abort "Error on processing site: #{site}" if site.nil?
+		host=host_tracker.url_2_host(site)
+		abort "Error on processing host: #{host}" if host.nil?
+		ip=host_tracker.local_host_2_ip(host)
+		ip=host_tracker.host_2_ip(host) if ip.nil?
+		next if ip.nil?
+		next unless host_tracker.is_ip?(ip)
+		port=host_tracker.url_2_port(site)
+		key=ip+":"
+		key+=port.to_s
+		@services[key]=true unless @services.key?(key)
+	end
+	f_site.close
+	host_tracker=nil
+end
+
+# Go through the new scan list and look up for known service from last quarter
+def diff
+	host_tracker=Wmap::HostTracker.new
+	f_new = File.open(ARGV[1],'r')
+	f_new.each do |line|
+		site=line.chomp.strip
+		site1=host_tracker.url_2_site(site)
+		abort "Error on processing site: #{site}" if site1.nil?
+		host=host_tracker.url_2_host(site1)
+		abort "Error on processing host: #{host}" if host.nil?
+		ip=host_tracker.local_host_2_ip(host)
+		ip=host_tracker.host_2_ip(host) if ip.nil?
+		abort "Error resolve host: #{host}" if ip.nil?
+		port=host_tracker.url_2_port(site1)
+		abort "Error retrieve service port on site: #{site}" if port.nil?
+		key=ip+":"
+		key+=port.to_s
+		if @services.key?(key)
+			puts "No"
+		else
+			puts "Yes"
+		end
+	end
+	f_new.close
+	host_tracker=nil 
+end
+
+parse_old
+diff

data/demos/filter_netinfo.rb

@@ -0,0 +1,23 @@
+# Exact netname and description from the whois query on an IP
+# Usage: ruby filter_netinfo.rb [file_ip] 
+require	"wmap"
+
+
+puts Wmap.banner
+whois = Wmap::Whois.new(:verbose=>false)
+#tracker = Wmap::CidrTracker.new
+
+#puts "IP ;  Netname ; Net Reference"
+f_ips = File.open(ARGV[0], 'r:iso-8859-1')
+f_ips.each do |line|
+	ip=line.chomp.split(',')[1]  
+	if whois.is_ip?(ip) or whois.is_cidr?(ip) 
+		netname=whois.get_netname(ip)
+		desc=whois.get_net_desc(ip)
+		#tr=tracker.track(ip)
+		puts "#{line.chomp};#{netname};#{desc}"
+	else
+		puts "#{line.chomp};;"
+	end
+end
+f_ips.close

data/demos/filter_prime.rb

@@ -0,0 +1,25 @@
+# Replace the embedded hostname within the url, based on the prime host table
+# Usage: ruby filter_prime.rb [file_host]
+require	"wmap"
+
+puts "URL | Prime URL"
+my_tracker=Wmap::HostTracker::PrimaryHost.new
+f_urls = File.open(ARGV[0], 'r')
+f_urls.each do |line|
+	url=line.chomp
+	if my_tracker.is_url?(url)
+		host=my_tracker.url_2_host(url)
+		ip=my_tracker.host_2_ip(host)
+		if my_tracker.ip_known?(ip)
+			p_host=my_tracker.local_ip_2_host(ip)
+			url_new=url.sub(host,p_host)
+		else
+			url_new=url
+		end
+		puts "#{url} | #{url_new}"
+	else
+		puts "#{url} | #{url}"
+	end
+end
+f_urls.close
+my_tracker=nil 

data/demos/filter_profiler.rb

@@ -0,0 +1,3 @@
+require "wmap"
+k=Wmap::NetworkProfiler.new(:verbose=>true)
+k.profile(ARGV[0])

data/demos/filter_redirection.rb

@@ -0,0 +1,19 @@
+# Internet domain fingerprint technique - print the redirection location if any
+# Usage: ruby filter_redirection.rb [file_urls] 
+require	"wmap"
+
+puts "List of URLs with Redirection:"
+puts "URL, Reponse Code, Redirection Location"
+myDis = Wmap::UrlChecker.new
+myDis.http_timeout=5000
+f_urls = File.open(ARGV[0], 'r')
+f_urls.each do |line|
+	url=line.chomp
+	host=myDis.url_2_host(url)
+	code=myDis.response_code(url)
+	if code >= 300 && code < 400
+		location=myDis.redirect_location(url)
+	end
+	puts "#{url}, #{code}, #{location}"
+end
+f_urls.close

data/demos/filter_site.rb

@@ -0,0 +1,40 @@
+# filter to select findings for the list of sites
+# Usage: ruby filter_site.rb [file targets] [file finding keys]
+# Input A is a list of target sites, input B is a list of site index keys for the finding summary
+# Output is a list of key for the matching purpose
+
+require "wmap"
+@map=Hash.new
+
+def build_map (file)
+	k=Wmap::SiteTracker.new
+	f=File.open(file,'r')
+	f.each do |line|
+		url=line.chomp.strip.downcase
+		if k.is_url?(url)
+			@map[k.url_2_site(url)]=true
+		else
+			#puts url
+		end
+	end
+	f.close
+	k=nil
+end
+
+
+build_map(ARGV[0])
+s=Wmap::SiteTracker.new
+f=File.open(ARGV[1],'r')
+f.each do |line|
+	url=line.chomp.strip.downcase
+	if s.is_url?(url)
+		site=s.url_2_site(url)
+		if @map.key?(site)
+			puts "yes"
+		else
+			puts "no"
+		end
+	else
+		puts "Invalid Internet URL"
+	end
+end

data/demos/filter_siteip.rb

@@ -0,0 +1,31 @@
+# Sample Site IP Tracker - Given a IP, flag it if not found in the site data repository
+# Usage: ruby filter_siteip.rb [file_web_hosts]
+require	"wmap"
+
+def known?(ip)
+	ip=ip.chomp.strip
+	myDis = Wmap::SiteTracker.new
+	known=false
+	if myDis.is_ip?(ip)
+		if myDis.siteip_known?(ip)
+			myDis=nil
+			return true
+		end
+	end
+	myDis=nil
+	return known
+end
+
+puts "Site IP, Status"
+
+
+f_ips = File.open(ARGV[0], 'r')
+f_ips.each do |line|
+	ip=line.chomp.strip
+	if known?(ip)
+		# do nothing
+	else
+		puts "#{ip}, unknown"
+	end
+end
+f_ips.close

data/demos/filter_status.rb

@@ -0,0 +1,17 @@
+# Print the URL of a site if it response to the HTTP request
+# Usage: ruby filter_status.rb [file_url_links] 
+require	"wmap"
+
+puts "List of URLs with the Valid Response Code:"
+puts "URL, Reponse Code, URL Finger Print, IP, Trusted CIDR, Trusted CIDR Netname"
+myDis = Wmap::UrlChecker.new(:verbose=>false, :http_timeout=>5000)
+
+f_urls = File.open(ARGV[0], 'r')
+f_urls.each do |line|
+	next if url.nil?
+	checker=myDis.check(url) 
+	host=line.chomp
+	tracker=Wmap.track(host)
+	puts "#{url}, #{checker['code']}, #{checker['md5']}, #{tracker['ip']}, #{tracker['cidr']}, #{tracker['netname']}"
+end
+f_urls.close

data/demos/filter_timestamp.rb

@@ -0,0 +1,23 @@
+# print out the timestamp for the sites
+# Usage: ruby filter_timestamp.rb [file_sites] 
+require	"wmap"
+
+tracker = Wmap::SiteTracker.instance
+
+puts "Site | Timestamp"
+f_urls = File.open(ARGV[0], 'r')
+f_urls.each do |line|
+	url=line.chomp  
+	if tracker.is_url?(url)  
+		site=tracker.url_2_site(url)
+		if tracker.site_known?(site)
+			ts=tracker.known_sites[site]['timestamp']
+			puts "#{site}|#{ts}"
+		else
+			puts site
+		end
+	else
+		puts url
+	end
+end
+f_urls.close

data/demos/filter_url.rb

@@ -0,0 +1,19 @@
+# filter to detect unknown internet domain
+# Input is a list of URLs
+# Output is an internet domain list that not currently tracked by the domain tracker
+
+require "wmap"
+
+k=Wmap::DomainTracker.new
+#k.verbose=true
+f=File.open(ARGV[0],'r')
+f.each do |line|
+	url=line.chomp.strip.downcase
+	host=k.url_2_host(url)
+	root=k.get_domain_root(host)
+	unless k.domain_known?(root)
+		puts root
+	end
+end
+f.close
+k=nil