threatinator 0.1.6 → 0.2.0

data/feeds/infiltrated_vabl-ip_reputation.feed

@@ -0,0 +1,29 @@
+provider "infiltrated"
+name "vabl_ip_reputation"
+fetch_http('http://www.infiltrated.net/vabl.txt')
+
+filter_whitespace
+filter_comments
+
+parse_eachline(:separator => "\n") do |event_generator, record|
+  fields = record.data.split(/ \| /)
+  next if record.nil?
+  
+  ip = fields[0].strip
+  # reason = fields[1].strip
+  # source = fields[2].strip
+  # detectdate = fields[3].strip
+  # sourcemd5 = fields[4].strip
+  # asn = fields[5].strip
+  # prefix = fields[6].strip
+  # orgname = fields[7].strip
+  # country = fields[8].strip
+  # domain = fields[9].strip
+  # asname = fields[19].strip
+  
+  event_generator.call() do |event|
+    event.type = :scanning
+    event.add_ipv4(ip) do |ipv4_event|
+    end
+  end
+end

data/feeds/isc_suspicious_high-domain_reputation.feed

@@ -0,0 +1,25 @@
+provider "isc"
+name "suspicious_high_domain_reputation"
+fetch_http('https://isc.sans.edu/feeds/suspiciousdomains_High.txt')
+
+feed_re = /^(?<domain>.*)/
+
+filter_whitespace
+filter_comments
+
+# Filter out IPv6 addresses
+filter do |record|
+  (record.data =~ /^Site/)
+  end
+
+parse_eachline(:separator => "\n") do |event_generator, record|
+  m = feed_re.match(record.data)
+  next if m.nil?
+
+  domain = m[:domain].strip
+  
+  event_generator.call() do |event|
+    event.type = :malware_host
+    event.add_fqdn(domain)
+  end
+end

data/feeds/isc_suspicious_low-domain_reputation.feed

@@ -0,0 +1,25 @@
+provider "isc"
+name "suspicious_low_domain_reputation"
+fetch_http('https://isc.sans.edu/feeds/suspiciousdomains_Low.txt')
+
+feed_re = /^(?<domain>.*)/
+
+filter_whitespace
+filter_comments
+
+# Filter out IPv6 addresses
+filter do |record|
+  (record.data =~ /^Site/)
+  end
+
+parse_eachline(:separator => "\n") do |event_generator, record|
+  m = feed_re.match(record.data)
+  next if m.nil?
+
+  domain = m[:domain].strip
+  
+  event_generator.call() do |event|
+    event.type = :malware_host
+    event.add_fqdn(domain)
+  end
+end

data/feeds/isc_suspicious_medium-domain_reputation.feed

@@ -0,0 +1,25 @@
+provider "isc"
+name "suspicious_medium_domain_reputation"
+fetch_http('https://isc.sans.edu/feeds/suspiciousdomains_Medium.txt')
+
+feed_re = /^(?<domain>.*)/
+
+filter_whitespace
+filter_comments
+
+# Filter out IPv6 addresses
+filter do |record|
+  (record.data =~ /^Site/)
+  end
+
+parse_eachline(:separator => "\n") do |event_generator, record|
+  m = feed_re.match(record.data)
+  next if m.nil?
+
+  domain = m[:domain].strip
+  
+  event_generator.call() do |event|
+    event.type = :malware_host
+    event.add_fqdn(domain)
+  end
+end

data/feeds/malwaredomainlist-url_reputation.feed

@@ -0,0 +1,16 @@
+provider "malwaredomainlist"
+name "url_reputation"
+fetch_http('http://www.malwaredomainlist.com/updatescsv.php')
+
+parse_csv(:headers => [:detectdate, :url, :ip, :rdns, :comment, :registrar_contact, :asn]) do |event_generator, record|
+  event_generator.call do |event|
+    event.type = :malware_host
+    if record.data[:url] == '-' then
+	  url = "http://"+record.data[:ip]
+	else
+      url = "http://"+record.data[:url]
+	end
+    event.add_url(url) do |url_event|
+    end
+  end
+end

data/feeds/malwaredomains-domain_reputation.feed

@@ -0,0 +1,27 @@
+provider "malwaredomains"
+name "domain_reputation"
+fetch_http('http://mirror1.malwaredomains.com/files/domains.txt')
+
+filter_whitespace
+filter_comments
+
+parse_eachline(:separator => "\n") do |event_generator, record|
+  fields = record.data.split(/\t/)
+  next if record.nil?
+  
+  # null_string  = fields[0]
+  # expires = fields[1]
+  remote_domain = fields[2]
+  # description = fields[3]
+  # source = fields[4]
+  # detecttime1 = fields[5]
+  # detecttime2 = fields[6]
+  # detecttime3 = fields[7]
+  # detecttime4 = fields[8]
+  # relisted = fields[9]
+  
+  event_generator.call() do |event|
+    event.type = :malware_host
+    event.add_fqdn(remote_domain)
+    end
+end

data/feeds/malwaredomains_dyndns-domain_reputation.feed

@@ -0,0 +1,27 @@
+provider "malwaredomains"
+name "dyndns_domain_reputation"
+fetch_http('http://mirror2.malwaredomains.com/files/dynamic_dns.txt')
+
+filter_whitespace
+filter_comments
+
+parse_eachline(:separator => "\n") do |event_generator, record|
+  fields = record.data.split(/\t/)
+  next if record.nil?
+  
+  remote_domain = fields[0]
+  # This is a badly formatted feed, below fields won't work
+  # source = fields[1]
+  # description = fields[2]
+  
+  # Some domains are conjoined with a domain#description
+  if remote_domain =~ /#/
+    remote_domain.gsub!(/#.*/, '')
+    end
+  remote_domain.strip!
+  
+  event_generator.call() do |event|
+    event.type = :scanning
+    event.add_fqdn(remote_domain)
+    end
+end

data/feeds/malwaredomains_justdomains-domain_reputation.feed

@@ -0,0 +1,18 @@
+provider "malwaredomains"
+name "justdomains_domain_reputation"
+fetch_http('http://mirror3.malwaredomains.com/files/justdomains')
+
+feed_re = /^(?<domain>.*)/
+
+filter_whitespace
+filter_comments
+
+parse_eachline(:separator => "\n") do |event_generator, record|
+  m = feed_re.match(record.data)
+  next if m.nil?
+
+  event_generator.call() do |event|
+    event.type = :malware_host
+    event.add_fqdn(m[:domain])
+  end
+end

data/feeds/multiproxy-ip_reputation.feed

@@ -0,0 +1,20 @@
+provider "multiproxy"
+name "ip_reputation"
+fetch_http('http://multiproxy.org/txt_all/proxy.txt')
+
+filter_whitespace
+filter_comments
+
+parse_eachline(:separator => "\n") do |event_generator, record|
+  fields = record.data.split(/\:/)
+  next if record.nil?
+  
+  ip = fields[0]
+  # port = fields[1].strip
+  
+  event_generator.call() do |event|
+    event.type = :scanning
+    event.add_ipv4(ip) do |ipv4_event|
+    end
+  end
+end

data/feeds/openphish-url_reputation.feed

@@ -0,0 +1,22 @@
+provider "openphish"
+name "url_reputation"
+fetch_http('http://openphish.com/feed.txt')
+
+filter_whitespace
+filter_comments
+
+# The last record I see is http://www. so filter that out
+filter do |record|
+  (record.data =~ /\/\/www\.$/)
+  end
+
+parse_eachline(:separator => "\n") do |event_generator, record|
+  url = record.data.strip()
+  next if url.nil?
+
+  event_generator.call() do |event|
+    event.type = :malware_host
+    event.add_url(url) do |url_event|
+    end
+  end
+end

data/feeds/packetmail_perimeterbad-ip_reputation.feed

@@ -0,0 +1,26 @@
+provider "packetmail"
+name "perimeterbad_ip_reputation"
+fetch_http('https://www.packetmail.net/iprep_perimeterbad.txt')
+
+filter_whitespace
+filter_comments
+
+parse_eachline(:separator => "\n") do |event_generator, record|
+  fields = record.data.split(/\t/)
+  next if record.nil?
+  
+  # date_time = fields[0]
+  remote_ip = fields[1]
+  # server_name = fields[2]
+  # status = fields[3]
+  # request = fields[4]
+  # http_referer = fields[5]
+  # user_agent = fields[6]
+  # day = fields[7]
+
+  event_generator.call() do |event|
+    event.type = :scanning
+    event.add_ipv4(remote_ip) do |ipv4_event|
+    end
+  end
+end

data/feeds/phishtank.feed

@@ -6,9 +6,9 @@ fetch_http('http://data.phishtank.com/data/online-valid.json.gz')
 extract_gzip
 parse_json() do |event_generator, record|
   event_generator.call do |event|
-    # TODO: parse URL
     # TODO: parse dates
     
+    event.add_url(record.data["url"])
     event.type = :phishing
     record.data["details"].each do |detail|
       if ip = detail["ip_address"]

data/feeds/sigmaproject_atma.feed

@@ -0,0 +1,25 @@
+provider "sigmaproject"
+name "atma_ip_reputation"
+
+fetch_http('https://blocklist.sigmaprojects.org/api.cfc?method=getList&lists=atma')
+
+# Filter out ip's
+filter do |record|
+  !(record.data =~ /\/32/)
+end
+
+extract_gzip
+parse_eachline() do |event_generator, record|
+
+   # We're tossing anything that's not a /32
+
+   ip = record.data
+   ip.gsub!(/\/32/, '')
+   ip.strip!
+   
+   event_generator.call() do |event|
+      event.type = :scanning
+      event.add_ipv4(ip) do |ipv4_event|
+	  end
+   end
+end

data/feeds/sigmaproject_spyware.feed

@@ -0,0 +1,24 @@
+provider "sigmaproject"
+name "spyware_ip_reputation"
+
+fetch_http('https://blocklist.sigmaprojects.org/api.cfc?method=getList&lists=spyware')
+extract_gzip
+
+# Filter out ip's
+filter do |record|
+  # TODO handle subnets
+  !(record.data =~ /\/32/)
+end
+
+parse_eachline() do |event_generator, record|
+
+   ip = record.data
+   ip.gsub!(/\/32/, '')
+   ip.strip!
+
+   event_generator.call() do |event|
+      event.type = :c2
+      event.add_ipv4(ip) do |ipv4_event|
+	  end
+   end
+end

data/feeds/sigmaproject_webexploit.feed

@@ -0,0 +1,26 @@
+provider "sigmaproject"
+name "webexploit_ip_reputation"
+
+fetch_http('https://blocklist.sigmaprojects.org/api.cfc?method=getList&lists=webexploit')
+extract_gzip
+
+# Filter out ip's
+filter do |record|
+  # TODO handle subnets
+  !(record.data =~ /\/32/)
+end
+
+parse_eachline() do |event_generator, record|
+
+   # We're tossing anything that's not a /32
+
+   ip = record.data
+   ip.gsub!(/\/32/, '')
+   ip.strip!
+   
+   event_generator.call() do |event|
+      event.type = :attacker
+      event.add_ipv4(ip) do |ipv4_event|
+	  end
+   end
+end

data/feeds/snort_bpf-ip_reputation.feed

@@ -0,0 +1,19 @@
+provider "snort"
+name "bpf_ip_reputation"
+fetch_http('http://labs.snort.org/feeds/ip-filter.blf')
+
+feed_re = /(?<ip>\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})/
+
+filter_whitespace
+filter_comments
+
+parse_eachline(:separator => "\n") do |event_generator, record|
+  m = feed_re.match(record.data)
+  next if m.nil?
+
+  event_generator.call() do |event|
+    event.type = :scanning
+    event.add_ipv4(m[:ip]) do |ipv4_event|
+    end
+  end
+end

data/feeds/steeman-ip_reputation.feed

@@ -0,0 +1,19 @@
+provider "steeman"
+name "ip_reputation"
+fetch_http('http://jeroen.steeman.org/FS-PlainText')
+
+feed_re = /(?<ip>\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})/
+
+filter_whitespace
+filter_comments
+
+parse_eachline(:separator => "\n") do |event_generator, record|
+  m = feed_re.match(record.data)
+  next if m.nil?
+
+  event_generator.call() do |event|
+    event.type = :scanning
+    event.add_ipv4(m[:ip]) do |ipv4_event|
+    end
+  end
+end

data/feeds/trustedsec-ip_reputation.feed

@@ -0,0 +1,17 @@
+provider "trustedsec"
+name "ip_reputation"
+fetch_http('https://www.trustedsec.com/banlist.txt')
+
+filter_whitespace
+filter_comments
+
+parse_eachline(:separator => "\n") do |event_generator, record|
+  next if record.nil?
+  ip = record.data.strip
+  
+  event_generator.call() do |event|
+    event.type = :scanning
+    event.add_ipv4(ip) do |ipv4_event|
+    end
+  end
+end

data/feeds/virbl-ip_reputation.feed

@@ -0,0 +1,24 @@
+provider "virbl"
+name "ip_reputation"
+fetch_http('http://virbl.org/download/virbl.dnsbl.bit.nl.txt')
+
+feed_re = /^(?<ip>\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})/
+
+filter_whitespace
+filter_comments
+
+# Filter out ip's
+filter do |record|
+  !(record.data =~ /^\d/)
+  end
+
+parse_eachline(:separator => "\n") do |event_generator, record|
+  m = feed_re.match(record.data)
+  next if m.nil?
+
+  event_generator.call() do |event|
+    event.type = :malware_host
+    event.add_ipv4(m[:ip]) do |ipv4_event|
+    end
+  end
+end