opendns-dnsdb 0.1.0

data/lib/opendns-dnsdb/dnsdb/label.rb

@@ -0,0 +1,105 @@
+
+require_relative 'siphash'
+
+module OpenDNS
+  class DNSDB
+    module Label
+      CACHE_KEY = 'Umbrella/OpenDNS'      
+
+      def distinct_labels(labels)
+        return [ labels ] unless labels.kind_of?(Hash)
+        Response::Distinct.new(labels.values.flatten.uniq)
+      end     
+      
+      def labels_by_name(names)
+        names_is_array = names.kind_of?(Enumerable)
+        names = [ names ] unless names_is_array
+        multi = Ethon::Multi.new
+        names_json = MultiJson.dump(names)
+        cacheid = SipHash::digest(CACHE_KEY, names_json).to_s(16)
+        url = "/infected/names/#{cacheid}.json"
+        query = query_handler(url, :get, { body: names_json })
+        multi.add(query)
+        multi.perform
+        responses = MultiJson.load(query.response_body)
+        responses = responses['scores']
+        responses.each_pair do |name, label|
+          responses[name] = [:suspicious, :unknown, :benign][label + 1]
+        end
+        responses = Response::HashByName[responses]
+        responses = responses.values.first unless names_is_array
+        responses
+      end
+      
+      def distinct_labels_by_name(names)
+        distinct_labels(labels_by_name(names))
+      end
+      
+      def include_suspicious?(names)
+        distinct_labels_by_name(names).include?(:suspicious)
+      end
+      
+      def is_suspicious?(name)
+        include_suspicious?(name)
+      end
+      
+      def include_benign?(names)
+        distinct_labels_by_name(names).include?(:benign)
+      end
+      
+      def is_benign?(name)
+        include_benign?(name)
+      end
+      
+      def include_unknown?(names)
+        distinct_labels_by_name(names).include?(:unknown)
+      end
+      
+      def is_unknown?(name)
+        include_unknown?(name)
+      end
+      
+      def suspicious_names(names)
+        labels = labels_by_name(names)
+        labels = [ labels ] unless labels.kind_of?(Enumerable)
+        names = labels.select { |name, label| label == :suspicious }.keys
+        Response::Distinct.new(names)
+      end
+      
+      def not_suspicious_names(names)
+        labels = labels_by_name(names)
+        labels = [ labels ] unless labels.kind_of?(Enumerable)
+        names = labels.select { |name, label| label != :suspicious }.keys
+        Response::Distinct.new(names)
+      end
+      
+      def unknown_names(names)
+        labels = labels_by_name(names)
+        labels = [ labels ] unless labels.kind_of?(Enumerable)
+        names = labels.select { |name, label| label == :unknown }.keys        
+        Response::Distinct.new(names)
+      end
+      
+      def not_unknown_names(names)
+        labels = labels_by_name(names)
+        labels = [ labels ] unless labels.kind_of?(Enumerable)
+        names = labels.select { |name, label| label != :unknown }.keys
+        Response::Distinct.new(names)
+      end
+      
+      def benign_names(names)
+        labels = labels_by_name(names)
+        labels = [ labels ] unless labels.kind_of?(Enumerable)
+        names = labels.select { |name, label| label == :benign }.keys
+        Response::Distinct.new(names)
+      end
+      
+      def not_benign_names(names)
+        labels = labels_by_name(names)
+        labels = [ labels ] unless labels.kind_of?(Enumerable)
+        names = labels.select { |name, label| label != :benign }.keys
+        Response::Distinct.new(names)
+      end            
+    end
+  end
+end

data/lib/opendns-dnsdb/dnsdb/related.rb

@@ -0,0 +1,92 @@
+
+require_relative 'rrutils'
+
+module OpenDNS
+  class DNSDB
+    module Related
+      include OpenDNS::DNSDB::RRUtils
+
+      def related_names_with_score(names, &filter)
+        names_is_array = names.kind_of?(Enumerable)
+        names = [ names ] unless names_is_array
+        multi = Ethon::Multi.new
+        queries_links = { }
+        queries_coocs = { }
+        names.each do |name|
+          url_links = "/links/name/#{name}.json"
+          query_links = query_handler(url_links)
+          multi.add(query_links)
+          queries_links[name] = query_links
+
+          url_coocs = "/recommendations/name/#{name}.json"
+          query_coocs = query_handler(url_coocs)
+          multi.add(query_coocs)
+          queries_coocs[name] = query_coocs
+        end
+        multi.perform
+        responses = { }
+        queries_coocs.each_pair do |name, query|
+          if query.response_body.empty?
+            responses[name] ||= { }
+            next
+          end
+          obj = MultiJson.load(query.response_body)
+          if pfs2 = Response::Raw.new(obj).pfs2
+            responses[name] = Hash[*pfs2.flatten]
+          else
+            responses[name] = { }
+          end
+        end
+        queries_links.each_pair do |name, query|
+          responses[name] ||= { }          
+          if query.response_body.empty?
+            next
+          end
+          obj = MultiJson.load(query.response_body)          
+          if tb1 = Response::Raw.new(obj).tb1          
+            upper = [1.0, tb1.map { |x| x[1] }.max].max
+            responses[name] = Hash[*tb1.map { |x| [x[0], x[1] / upper] }.flatten]
+          end
+        end
+        responses = Response::HashByName[responses]
+        if block_given?
+          responses.each_key do |name|
+            responses[name].select! do |related_name, score|
+              filter.call(related_name, score)
+            end
+          end
+        end
+        responses = responses.values.first unless names_is_array
+        responses
+      end
+      
+      def related_names(names, options = { }, &filter)
+        res = related_names_with_score(names, &filter) || { }
+        if names.kind_of?(Enumerable)
+          res.merge(res) do |name, v|
+            res0 = v.keys
+            res0 = res0[0...options[:max_names]] if options[:max_names]
+            res0
+          end          
+        else          
+          res0 = res.keys
+          res0 = res[0...options[:max_names]] if options[:max_names]
+          res0
+        end
+      end
+      
+      def distinct_related_names(names, options = { }, &filter)
+        res = Response::Distinct.new(distinct_rrs(related_names(names, &filter)))
+        res = res[0...options[:max_names]] if options[:max_names]
+        if (options[:max_depth] || 1) > 1
+          options0 = options.clone
+          options0[:max_depth] -= 1
+          related = distinct_related_names(res, options0, &filter)
+          res = (res + related).uniq
+          res = res[0...options[:max_names]] if options[:max_names]          
+        end
+        res
+      end
+    end
+  end
+end

data/lib/opendns-dnsdb/dnsdb/response.rb

@@ -0,0 +1,41 @@
+
+module OpenDNS
+  class DNSDB
+    module Response
+      module Common
+      end
+      
+      class Raw < Hashie::Mash
+        include Common
+        
+        def initialize(source_hash = nil, default = nil, &blk)
+          if source_hash
+            if source_hash[:first_seen]
+              source_hash[:first_seen] = Date.parse(source_hash[:first_seen])
+            end
+            if source_hash[:last_seen]
+              source_hash[:last_seen] = Date.parse(source_hash[:last_seen])
+            end
+          end
+          super(source_hash, default, &blk)
+        end
+      end
+      
+      class HashByName < Hash
+        include Common        
+      end
+      
+      class HashByIP < Hash
+        include Common
+      end
+      
+      class Distinct < Array
+        include Common
+      end
+      
+      class TimeSeries < Array
+        include Common
+      end
+    end
+  end
+end

data/lib/opendns-dnsdb/dnsdb/rrutils.rb

@@ -0,0 +1,11 @@
+
+module OpenDNS
+  class DNSDB
+    module RRUtils
+      def distinct_rrs(rrs)
+        return rrs unless rrs.kind_of?(Hash)
+        Response::Distinct.new(rrs.values.flatten.uniq)
+      end
+    end
+  end
+end

data/lib/opendns-dnsdb/dnsdb/siphash.rb

@@ -0,0 +1,94 @@
+module SipHash
+
+  def self.digest(key, msg)
+    s = State.new(key)
+    len = msg.size
+    iter = len / 8
+
+    iter.times do |i|
+      m = msg.slice(i * 8, 8).unpack("Q<")[0]
+      s.apply_block(m)
+    end
+
+    m = last_block(msg, len, iter)
+
+    s.apply_block(m)
+    s.finalize
+    s.digest
+  end
+
+  private
+
+  def self.last_block(msg, len, iter)
+    last = (len << 56) & State::MASK_64;
+      
+    r = len % 8
+    off = iter * 8
+
+    last |= msg[off + 6].ord << 48 if r >= 7
+    last |= msg[off + 5].ord << 40 if r >= 6
+    last |= msg[off + 4].ord << 32 if r >= 5
+    last |= msg[off + 3].ord << 24 if r >= 4
+    last |= msg[off + 2].ord << 16 if r >= 3
+    last |= msg[off + 1].ord << 8 if r >= 2
+    last |= msg[off].ord if r >= 1
+    last
+  end
+
+  class State
+    
+    MASK_64 = 0xffffffffffffffff
+
+    def initialize(key)
+      @v0 = 0x736f6d6570736575
+      @v1 = 0x646f72616e646f6d
+      @v2 = 0x6c7967656e657261
+      @v3 = 0x7465646279746573
+
+      k0 = key.slice(0, 8).unpack("Q<")[0]
+      k1 = key.slice(8, 8).unpack("Q<")[0]
+
+      @v0 ^= k0
+      @v1 ^= k1
+      @v2 ^= k0
+      @v3 ^= k1
+    end
+
+    def apply_block(m)
+      @v3 ^= m
+      2.times { compress }
+      @v0 ^= m
+    end
+
+    def rotl64(num, shift)
+      ((num << shift) & MASK_64) | (num >> (64 - shift))
+    end
+
+    def compress
+      @v0 = (@v0 + @v1) & MASK_64
+      @v2 = (@v2 + @v3) & MASK_64 
+      @v1 = rotl64(@v1, 13)
+      @v3 = rotl64(@v3, 16)
+      @v1 ^= @v0
+      @v3 ^= @v2
+      @v0 = rotl64(@v0, 32)
+      @v2 = (@v2 + @v1) & MASK_64
+      @v0 = (@v0 + @v3) & MASK_64
+      @v1 = rotl64(@v1, 17)
+      @v3 = rotl64(@v3, 21)
+      @v1 ^= @v2
+      @v3 ^= @v0
+      @v2 = rotl64(@v2, 32)
+    end
+
+    def finalize
+      @v2 ^= 0xff
+      4.times { compress }
+    end
+
+    def digest
+      @v0 ^ @v1 ^ @v2 ^ @v3
+    end
+
+  end
+end

data/lib/opendns-dnsdb/dnsdb/traffic.rb

@@ -0,0 +1,80 @@
+
+module OpenDNS
+  class DNSDB
+    module Traffic
+      require 'cgi'
+      require 'date'
+      require 'descriptive_statistics'      
+      
+      DEFAULT_DAYS_BACK = 7
+
+      def daily_traffic_by_name(names, options = { })
+        names_is_array = names.kind_of?(Enumerable)
+        names = [ names ] unless names_is_array
+        multi = Ethon::Multi.new
+        date_end = options[:start] || Date.today
+        date_end_s = CGI::escape("#{date_end.year}/#{date_end.month}/#{date_end.day}/23")
+        days_back = options[:days_back] || DEFAULT_DAYS_BACK
+        date_start = date_end - days_back
+        date_start += 1 unless date_start == date_end
+        date_start_s = CGI::escape("#{date_start.year}/#{date_start.month}/#{date_start.day}/00")
+        date_end, date_start = date_start, date_end if date_start > date_end
+        queries_traffic = { }
+        names.each do |name|
+          name0 = name.gsub(/^www[.]/, '')
+          url_traffic = "/appserver/?v=1&function=domain2-system&domains=#{name0}" +
+          "&locations=&start=#{date_start_s}&stop=#{date_end_s}"
+          query_traffic = query_handler(url_traffic)
+          multi.add(query_traffic)
+          queries_traffic[name] = query_traffic
+        end
+        multi.perform
+        responses = { }
+        queries_traffic.each_pair do |name, query|
+          obj = MultiJson.load(query.response_body)
+          tc = obj['response']
+          tc = tc.group_by { |x| x[0].split('/')[0...3].join('/') }
+          tc.each_key do |date_s|
+            tc[date_s] = tc[date_s].inject(0) { |a, x| a + x[1] }
+          end
+          responses[name] = tc.values
+        end
+        responses = Response::HashByName[responses]
+        responses = responses.values.first unless names_is_array
+        responses        
+      end
+
+      def relative_standard_deviation(names_ts)
+        names_ts_is_hash = names_ts.kind_of?(Hash)
+        names_ts = { x: names_ts } unless names_ts_is_hash
+        responses = { }
+        names_ts.each_pair do |name, ts|
+          mean = ts.mean
+          if mean <= 0.0
+            responses[name] = 0.0
+          else
+            responses[name] = ts.standard_deviation / mean * 100.0
+          end
+        end
+        responses = Response::HashByName[responses]
+        responses = responses.values.first unless names_ts_is_hash
+        responses
+      end
+      
+      def high_pass_filter(names_ts, options)
+        names_ts_is_hash = names_ts.kind_of?(Hash)
+        names_ts = { x: names_ts } unless names_ts_is_hash
+        responses = { }
+        cutoff = options[:cutoff]
+        names_ts.each_pair do |name, ts|
+          responses[name] = ts.map { |x| x < cutoff ? 0.0 : x }
+        end
+        responses = Response::HashByName[responses]
+        responses = responses.values.first unless names_ts_is_hash
+        responses        
+      end
+    end
+  end
+end
+
+      

data/lib/opendns-dnsdb/version.rb

@@ -0,0 +1,5 @@
+module OpenDNS
+  class DNSDB
+    VERSION = '0.1.0'
+  end
+end

data/opendns-dnsdb.gemspec

@@ -0,0 +1,20 @@
+# -*- encoding: utf-8 -*-
+require File.expand_path('../lib/opendns-dnsdb/version', __FILE__)
+
+Gem::Specification.new do |gem|
+  gem.authors       = ["Frank Denis"]
+  gem.email         = ["frank@opendns.com"]
+  gem.description   = "Client library for the OpenDNS Security Graph"
+  gem.summary       = gem.description
+  gem.homepage      = "https://github.com/jedisct1/opendns-dnsdb-ruby"
+  
+  gem.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  gem.files         = `git ls-files`.split("\n")
+  gem.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
+  gem.name          = "opendns-dnsdb"
+  gem.require_paths = ["lib"]
+  gem.version       = OpenDNS::DNSDB::VERSION
+
+  gem.add_development_dependency 'rake'
+  gem.add_development_dependency 'rspec', '>= 2.14'
+end