threatinator 0.1.6 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 9094ef4d662acb87c6f8425a90700527449fa7e6
-  data.tar.gz: ed120159fc6800df5902e105634c006844707f31
+  metadata.gz: 49782ab8639faa3f2f202e056f780b6fdeff09df
+  data.tar.gz: d2f4fb0ed1056f32f6509a9b8f1a9740170cc2c1
 SHA512:
-  metadata.gz: c98dc47be2f5aabc58a776af93401cd1b01ef380216992fd9310794f08c21e63a03b212081ce1cd64e50b25022c7d8ea61a87fb693b76b8795200745b5a92c3c
-  data.tar.gz: a8a459c5103636ba1c45c500bafb650162145ef01897148a5c54e6e4e9db4f7960de94868159e91964cccd5042607b3b554096f4c1c35e74cf4fa59b534b9b7d
+  metadata.gz: 4e4b0f2c3686572067acee715834addb78cd66fc7d718a5208506fc9ae9b676037ae13d18b9bf2e33d4a9230a345aab00ad50ba1841b92f1d154be5f3800fd01
+  data.tar.gz: c3b15041ad98b6da912de32b90e540a9634572222158d62503734c324d0eec3870b763d371b16abf88d413b74c6d412899bf4e4d3dd8e006a5d32eaa6d593f40

data/CHANGELOG.md

@@ -3,15 +3,26 @@ Next
 
 * Your contribution here.
 
+0.2.0
+====
+
+* Fix file-handle leak - [@justfalter](https://github.com/justfalter)
+* BREAKING CHANGE: Event#ipv4s -> now an Ipv4ObservableCollection that consists on Ipv4Observable objects. 
+* Normalize and validate ipv4 addresses.
+* Add URL support [@justfalter](https://github.com/justfalter)
+* Re-implement Event model, get rid of PropertyDefiner [@justfalter](https://github.com/justfalter)
+* Disable decoding when testing the parsing of a single record. [@justfalter](https://github.com/justfalter)
+
 0.1.6
 ====
 
-* Add JSON support: 'threatinator list --list.format=json'  - [@justfalter](https://github.com/justfalter)
+* [#121](https://github.com/cikl/threatinator/issues/121) Add threatinator list 'json' output format. [@justfalter](https://github.com/justfalter)
 
 0.1.5
 ====
 
-* Add missing requrie for set. Fixes #115. - [@justfalter](https://github.com/justfalter)
+* [#115](https://github.com/cikl/threatinator/issues/56): Add missing require for 'set' - [@justfalter](https://github.com/justfalter)
+
 0.1.2
 ====
 

data/Gemfile

@@ -1,11 +1,16 @@
 # A sample Gemfile
 source "https://rubygems.org"
 
-gem 'typhoeus', '~> 0.6'
-gem 'nokogiri', '~> 1.6'
-gem 'docile', '~> 1.1'
-gem 'virtus', '~> 1.0'
-gem 'gli', '~> 2.12'
+gem 'typhoeus', '~> 0.6.0'
+gem 'nokogiri', '~> 1.6.0'
+gem 'docile', '~> 1.1.0'
+gem 'virtus', '~> 1.0.0'
+gem 'gli', '~> 2.12.0'
+gem 'activemodel', '~> 4.0'
+gem 'equalizer', '> 0.0.0'
+gem 'addressable', '~> 2.3.0'
+gem 'log4r', '~> 1.1.10'
+gem 'ruby-ip', '~> 0.9.3'
 
 platforms :mingw, :mswin, :ruby do
   gem 'oj', '~> 2.9'
@@ -13,16 +18,16 @@ end
 
 group :test do
   gem 'rake'
-  gem 'multi_json', '~> 1.10'
-  gem 'simplecov', '~> 0.9'
+  gem 'multi_json', '~> 1.10.0'
+  gem 'simplecov', '~> 0.9.0'
   gem 'simplecov-vim', '= 0.0.1'
-  gem 'rspec', '~> 3'
-  gem 'rspec-its', '~> 1'
-  gem 'webmock', '~> 1'
-  gem 'factory_girl', '~> 4'
+  gem 'rspec', '~> 3.0'
+  gem 'rspec-its', '~> 1.0'
+  gem 'webmock', '~> 1.0'
+  gem 'factory_girl', '~> 4.0'
 end
 
 group :development do
-  gem 'jeweler', '~> 2'
-  gem 'pry', '~> 0.10'
+  gem 'jeweler', '~> 2.0'
+  gem 'pry', '~> 0.10.0'
 end

data/Rakefile

@@ -39,7 +39,7 @@ begin
 rescue LoadError
 else
   RSpec::Core::RakeTask.new(:spec) do |spec|
-    spec.pattern = FileList['spec/**/*_spec.rb']
+    spec.pattern = 'spec/**/*_spec.rb'
   end
 end
 

data/VERSION

@@ -1 +1 @@
-0.1.6
+0.2.0

data/feeds/ET_block-ip_reputation.feed

@@ -0,0 +1,26 @@
+provider "emergingthreats"
+name "block_ip_reputation"
+fetch_http('http://rules.emergingthreats.net/fwrules/emerging-Block-IPs.txt')
+
+filter_whitespace
+filter_comments
+
+# 3 categories of IP listed here;
+# Shadowserver C2 IPs, which we want
+# Spamhaus drop nets, which are covered in another feed
+# Dshield top attackers which are also covered in another feed
+# The ones we don't want are all CIDR
+filter do |record|
+  (record.data =~ /\//)
+  end
+
+parse_eachline(:separator => "\n") do |event_generator, record|
+  ip = record.data.strip()
+  next if ip.nil?
+
+  event_generator.call() do |event|
+    event.type = :c2
+    event.add_ipv4(ip) do |ipv4_event|
+    end
+  end
+end

data/feeds/ET_openbadlist-ip_reputation.feed

@@ -0,0 +1,35 @@
+provider "emergingthreats"
+name "openbadlist_ip_reputation"
+fetch_http('https://raw.githubusercontent.com/EmergingThreats/et-open-bad-ip-list/master/IPs.txt')
+
+filter_whitespace
+filter_comments
+
+# Filter out ip's
+filter do |record|
+  # TODO Add handling for CIDR's once supported
+  !(record.data =~ /\/32/)
+end
+
+parse_eachline(:separator => "\n") do |event_generator, record|
+  fields = record.data.split(/; /)
+  next if record.nil?
+  
+  # date = fields[0]
+  ip = fields[1]
+  # description = fields[2]
+
+  # We're tossing anything that's not a /32
+  ip.gsub!(/\/32/, '')
+  ip.strip!
+  if ip =~ /\//
+    puts ip
+    next
+  end
+  
+  event_generator.call() do |event|
+    event.type = :c2
+    event.add_ipv4(ip) do |ipv4_event|
+    end
+  end
+end

data/feeds/bambenek_c2_masterlist-domain_reputation.feed

@@ -0,0 +1,15 @@
+provider "bambenek"
+name "c2_masterlist"
+fetch_http('http://osint.bambenekconsulting.com/feeds/c2-dommasterlist.txt')
+
+filter do |record|
+  record.data[:domain].start_with?("#")
+end
+
+parse_csv(:headers => [:domain, :description, :moreinfo]) do |event_generator, record|
+  event_generator.call do |event|
+    event.type = :c2
+    event.add_fqdn(record.data[:domain]) do |fqdn_event|
+    end
+  end
+end

data/feeds/bambenek_c2_masterlist-ip_reputation.feed

@@ -0,0 +1,15 @@
+provider "bambenek"
+name "c2_masterlist_ip"
+fetch_http('http://osint.bambenekconsulting.com/feeds/c2-ipmasterlist.txt')
+
+filter do |record|
+  record.data[:ip].start_with?("#")
+end
+
+parse_csv(:headers => [:ip, :description, :date, :moreinfo]) do |event_generator, record|
+  event_generator.call do |event|
+    event.type = :c2
+    event.add_ipv4(record.data[:ip]) do |ipv4_event|
+    end
+  end
+end

data/feeds/bambenek_dga_feed-domain_reputation.feed

@@ -0,0 +1,15 @@
+provider "bambenek"
+name "dga_feed"
+fetch_http('http://osint.bambenekconsulting.com/feeds/dga-feed.txt')
+
+filter do |record|
+  record.data[:domain].start_with?("#")
+end
+
+parse_csv(:headers => [:domain, :description, :moreinfo]) do |event_generator, record|
+  event_generator.call do |event|
+    event.type = :c2
+    event.add_fqdn(record.data[:domain]) do |fqdn_event|
+    end
+  end
+end

data/feeds/berkeley-ip_reputation.feed

@@ -0,0 +1,23 @@
+provider "berkeley"
+name "ip_reputation"
+fetch_http('https://security.berkeley.edu/aggressive_ips/ips')
+
+feed_re = /^HOSTILE_IP\: (?<ip>\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})\s*LAST_SEEN\: (?<last_seen>\d{10})/
+
+filter_whitespace
+filter_comments
+
+filter do |record|
+  !(record.data =~ /^HOSTILE_IP/)
+  end
+
+parse_eachline(:separator => "\n") do |event_generator, record|
+  m = feed_re.match(record.data)
+  next if m.nil?
+
+  event_generator.call() do |event|
+    event.type = :scanning
+    event.add_ipv4(m[:ip]) do |ipv4_event|
+    end
+  end
+end

data/feeds/bitcash_cz_blacklist.feed

@@ -0,0 +1,20 @@
+provider "bitcash_cz"
+name "blacklist"
+
+fetch_http('http://bitcash.cz/misc/log/blacklist')
+
+filter_whitespace
+filter_comments
+
+parse_eachline() do |event_generator, record|
+
+  fields = record.data.split(/\s/)
+  next if fields[0].nil?
+  ip = fields[0].strip()
+  
+  event_generator.call() do |event|
+    event.type = :scanning
+    event.add_ipv4(ip) do |ipv4_event|
+    end
+  end
+end

data/feeds/botscout-ip_reputation.feed

@@ -0,0 +1,24 @@
+provider "botscout"
+name "ip_reputation"
+fetch_http('http://botscout.com/last_caught_cache.htm')
+
+feed_re = /(?<ip>\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})/
+
+filter_whitespace
+filter_comments
+
+# Filter out anything that doesn't have an IP
+filter do |record|
+  !(record.data =~ /\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}/)
+  end
+
+parse_eachline(:separator => "\n") do |event_generator, record|
+  m = feed_re.match(record.data)
+  next if m.nil?
+
+  event_generator.call() do |event|
+    event.type = :scanning
+    event.add_ipv4(m[:ip]) do |ipv4_event|
+    end
+  end
+end

data/feeds/cert_mxpoison-ip_reputation.feed

@@ -0,0 +1,21 @@
+# https://www.cert.org/blogs/certcc/post.cfm?EntryID=206
+
+provider "cert"
+name "mxpoison_ip_reputation"
+fetch_http('http://www.cert.org/downloads/mxlist.ips.txt')
+
+feed_re = /(?<ip>\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})/
+
+filter_whitespace
+filter_comments
+
+parse_eachline(:separator => "\n") do |event_generator, record|
+  m = feed_re.match(record.data)
+  next if m.nil?
+
+  event_generator.call() do |event|
+    event.type = :malware_host
+    event.add_ipv4(m[:ip]) do |ipv4_event|
+    end
+  end
+end

data/feeds/chaosreigns-ip_reputation.feed

@@ -0,0 +1,36 @@
+provider "chaosreigns"
+name "ip_reputation"
+fetch_http('http://www.chaosreigns.com/iprep/iprep.txt')
+
+filter_whitespace
+filter_comments
+
+# Until we support listing arbitrary fields, strip out 100% ham scored items
+filter do |record|
+  (record.data =~ /\s100\s/)
+  end
+  
+# Until we support v6, strip them out
+filter do |record|
+  (record.data =~ /\:/)
+  end
+  
+# Last line is a NULL, strip it out
+filter do |record|
+  (record.data =~ /^\s*0\s*0$/)
+  end
+
+parse_eachline(:separator => "\n") do |event_generator, record|
+  fields = record.data.strip.split(/\s/)
+  next if record.nil?
+  
+  ip = fields[0].strip
+  # whitelist = fields[1].strip # ...the actual percentage of email from each IP which is ham...
+  # total_count_logo = fields[2].strip #...a count of the total emails from that IP (as a logarithm). 
+  
+  event_generator.call() do |event|
+    event.type = :scanning
+    event.add_ipv4(ip) do |ipv4_event|
+    end
+  end
+end

data/feeds/cydef_torexit-ip_reputation.feed

@@ -0,0 +1,24 @@
+provider "cydef"
+name "torexit_ip_reputation"
+fetch_http('https://cydef.us/torexit.txt')
+
+feed_re = /(?<ip>\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})/
+
+filter_whitespace
+filter_comments
+
+# Filter out IPv6 addresses and the timestamp
+filter do |record|
+  (record.data =~ /\:/)
+  end
+
+parse_eachline(:separator => "\n") do |event_generator, record|
+  m = feed_re.match(record.data)
+  next if m.nil?
+
+  event_generator.call() do |event|
+    event.type = :scanning
+    event.add_ipv4(m[:ip]) do |ipv4_event|
+    end
+  end
+end

data/feeds/danger_bruteforce-ip_reputation.feed

@@ -0,0 +1,23 @@
+provider "danger"
+name "bruteforce_ip_reputation"
+fetch_http('http://danger.rulez.sk/projects/bruteforceblocker/blist.php')
+
+filter_whitespace
+filter_comments
+
+parse_eachline(:separator => "\n") do |event_generator, record|
+  next if record.nil?
+  fields = record.data.split(/\s/)
+  
+  # ipv4 = fields[0]
+  # last_reported_date = fields[2]
+  # last_reported_time = fields[3]
+  # count = fields[4]
+  # count = fields[5]
+
+  event_generator.call() do |event|
+    event.type = :scanning
+    event.add_ipv4(fields[0]) do |ipv4_event|
+    end
+  end
+end

data/feeds/falconcrest-ip_reputation.feed

@@ -0,0 +1,18 @@
+provider "falconcrest"
+name "ip_reputation"
+fetch_http('http://www.falconcrest.eu/IPBL.aspx')
+
+parse_xml("//td/span") do |event_generator, record|
+  node = record.node
+  ip = node.text
+  next if ip.nil?
+  next if ip.empty?
+
+  ip.gsub!(/\(/, '')
+
+  event_generator.call() do |event|
+    event.type = :spamming
+    event.add_ipv4(ip) do |ipv4_event|
+    end
+  end
+end

data/feeds/h3x_asprox.feed

@@ -0,0 +1,17 @@
+provider "h3x"
+name "asprox"
+
+fetch_http('http://atrack.h3x.eu/api/asprox_all.php')
+
+parse_eachline() do |event_generator, record|
+
+  fields = record.data.split(/\:/)
+  ip = fields[0]
+  # port = fields[1]
+
+  event_generator.call() do |event|
+    event.type = :c2
+    event.add_ipv4(ip) do |ipv4_event|
+    end
+  end
+end

data/feeds/hosts-file_hphostspartial-domain_reputation.feed

@@ -0,0 +1,18 @@
+provider "hosts-file"
+name "hphostspartial_domain_reputation"
+fetch_http('http://hosts-file.net/hphosts-partial.txt')
+
+filter_whitespace
+filter_comments
+
+parse_eachline(:separator => "\n") do |event_generator, record|
+  fields = record.data.split(/\t/)
+  
+  # localhost = fields[0]
+  remote_domain = fields[1].strip
+  
+  event_generator.call() do |event|
+    event.type = :malware_host
+    event.add_fqdn(remote_domain)
+    end
+end