google-safe-browsing-plugin 0.1.1

data/lib/google/safe_browsing_update_helper.rb

@@ -0,0 +1,171 @@
+module Google
+  module SafeBrowsingUpdateHelper
+    
+    def update_local_shavar_info update_obj
+      gsb = safe_browsing_service
+      update_next_update_time(gsb, update_obj.next) if update_obj.next
+      if update_obj.reset
+        if Rails.configuration.respond_to?(:google_safe_browsing_upon_reset)
+           Rails.configuration.google_safe_browsing_upon_reset.call
+        else
+          Rails.logger.warn "I got a reset from Google... Don't know what to do."
+        end
+      end
+        
+      if update_obj.has_lists?
+        update_obj.lists.keys.each do |name|
+          if SafeBrowsingList.valid_list?(name)
+            list = SafeBrowsingList.find_by_name(name.to_s)
+            update_add_sub_chunks list, update_obj.get_ad_chunk_ids(name), SafeBrowsingShavar::CHUNK_TYPE_ADD
+            update_add_sub_chunks list, update_obj.get_sd_chunk_ids(name), SafeBrowsingShavar::CHUNK_TYPE_SUB
+          else
+            Rails.logger.info "Got invalid list name [#{name}]"
+          end
+        end
+      end
+    end
+  
+    def save_redirect_urls urls, list_obj
+      urls.each do |url|
+        obj = SafeBrowsingRedirectUrl.for_url_and_list_id(url, list_obj.id).first
+        if obj.nil?
+          obj = SafeBrowsingRedirectUrl.create(url: url, google_safe_browsing_list_id: list_obj.id)
+        end
+      end
+    end
+  
+    def update_redirect_urls url, list_obj, attributes
+      obj = SafeBrowsingRedirectUrl.for_url_and_list_id(url, list_obj.id).first
+      obj.update_attributes(attributes)
+    end
+    
+    def update_add_sub_chunks list_obj, del_chunk_ids, chunk_type
+      
+      del_chunk_ids.each do |chunk_id|
+        if chunk_id.is_a?(Range)
+          chunk.each do |chunk_id|
+            SafeBrowsingShavar.where(google_safe_browsing_list_id: list_obj.id, chunk_num: chunk_id, chunk_type: chunk_type).destroy_all
+          end
+        else
+          SafeBrowsingShavar.where(google_safe_browsing_list_id: list_obj.id, chunk_num: chunk_id, chunk_type: chunk_type).destroy_all
+        end
+      end
+    end
+
+    def update_shavar_chunk adds, subs, shavar_list
+      update_add_shavar_chunk adds, shavar_list
+      update_sub_shavar_chunk subs, shavar_list
+    end
+  
+    def update_add_shavar_chunk adds, shavar_list
+      adds.each do |add|
+        
+        chunk_num = add[:chunk_num]
+        add[:chunk_data].each do |host_key, prefixes|
+          if prefixes.empty?
+            shavar = SafeBrowsingShavar.where(
+                       google_safe_browsing_list_id: shavar_list.id, 
+                       chunk_type: SafeBrowsingShavar::CHUNK_TYPE_ADD, 
+                       chunk_num: chunk_num, host_key: host_key, prefix: nil).first_or_create
+          else
+            prefixes.each do |prefix|
+              shavar = SafeBrowsingShavar.where(
+                         google_safe_browsing_list_id: shavar_list.id, 
+                         chunk_type: SafeBrowsingShavar::CHUNK_TYPE_ADD, 
+                         chunk_num: chunk_num, host_key: host_key, prefix: prefix).first_or_create
+            end
+          end
+        end
+      end
+
+    end
+    
+    def update_sub_shavar_chunk subs, shavar_list
+      subs.each do |sub|
+        chunk_num = sub[:chunk_num]
+        
+        sub[:chunk_data].each do |host_key, chunk_num_hash_prefix|
+          chunk_num_hash_prefix.each do |add_chunk_num, prefixes|
+            if prefixes.empty?
+              shavar = SafeBrowsingShavar.where(
+                         google_safe_browsing_list_id: shavar_list.id, 
+                         chunk_type: SafeBrowsingShavar::CHUNK_TYPE_SUB, 
+                         chunk_num: chunk_num, host_key: host_key, 
+                         add_chunk_num: add_chunk_num, prefix: nil).first_or_create
+            else
+              prefixes.each do |prefix|
+                shavar = SafeBrowsingShavar.where(
+                           google_safe_browsing_list_id: shavar_list.id, 
+                           chunk_type: SafeBrowsingShavar::CHUNK_TYPE_SUB, 
+                           chunk_num: chunk_num, host_key: host_key, 
+                           add_chunk_num: add_chunk_num, prefix: prefix).first_or_create
+              end
+            end
+          end # chunk_num_hash_prefix.each
+        end # subs[:chunk_data].each
+      end # subs.each
+
+    end
+    
+    #
+    # chunk_id_arr: [113121, 113122, 113123, 113132], in increasing order
+    # return: "113121-113123,113132"
+    #
+    def gen_chunk_nums_string chunk_id_arr
+      ranges_and_integers = []
+      first = last = chunk_id_arr.shift
+      return "" if first.nil?
+      
+      increment = 0
+      while !(int = chunk_id_arr.shift).nil?
+        increment += 1
+        if int == first + increment
+          last = int
+          next
+        else
+          if first == last
+            ranges_and_integers << first
+          else
+            ranges_and_integers << (first..last)  
+          end
+
+          first = last = int
+          increment = 0
+        end
+      end
+      
+      if first == last
+        ranges_and_integers << first
+      else
+        ranges_and_integers << (first..last)  
+      end
+      
+      range_and_int_arr_to_string(ranges_and_integers)
+    end
+    
+    
+    def safe_browsing_service
+      Function.where(name: Function::GoogleSafeBrowsing).first
+    end
+  
+    def update_next_update_time link_function, ts
+      link_function.update_attributes(:next_updated_at => Time.now + ts)
+    end
+  
+
+    protected 
+    
+    def range_and_int_arr_to_string arr
+      ret = []
+      arr.each do |member|
+        if member.is_a?(Range)
+          ret << member.first.to_s + "-" + member.last.to_s
+        elsif member.is_a?(Integer)
+          ret << member.to_s
+        end
+      end
+      ret.join(",")
+    end
+  
+  end
+end

data/lib/google/sha_util.rb

@@ -0,0 +1,22 @@
+module Google
+  module ShaUtil
+    extend self
+    
+    SHA256 = OpenSSL::Digest::SHA256.new
+    
+    def sha256_hex str, prefix = nil
+      hash = sha256_digest(str).unpack("H64").first
+      if prefix
+        hash.first(prefix)  # first 'prefix' chars
+      else
+        hash
+      end
+    end
+    
+    def sha256_digest str
+      SHA256.digest(str)
+    end
+    
+    
+  end
+end

data/lib/google/url_canonicalizer.rb

@@ -0,0 +1,36 @@
+
+module Google
+  module UrlCanonicalizer
+    extend self
+    
+    ANCHOR_REGEX = /(?:#(?:[-\w~!$+|.,*:=]|%[A-Fa-f\d]{2})*)?\b/
+    URL_REGEX = /(?<protocol>(?:ht|f)tp(?:s?)\:\/\/|~\/|\/)?(?<user_pwd>\w+:\w+@)?(?<host>((?<sub>[-\w]+\.)+(?<top>com|org|net|gov|mil|biz|info|mobi|name|aero|jobs|museum|travel|[a-z]{2}))|(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}))(?<port>:[\d]{1,5})?(?<dir>(?:(?:\/(?:[-\w~!$+|.=]|%[A-Fa-f\d]{2}|#!)+)+|\/)+|\?|#)?(?<query>(?:\?(?:[-\w~!$+|.,*:]|%[A-Fa-f\d{2}])+=?(?:[-\w~!$+|.,*:=\/]|%[A-Fa-f\d]{2})*)(?:&(?:[-\w~!$+|.,*:]|%[A-Fa-f\d{2}])+=?(?:[-\w~!$+|.,*:=\/]|%[A-Fa-f\d]{2})*)*)*\b/
+    
+    def apply input_url
+      url = input_url.to_s.encode("ASCII-8BIT", :invalid => :replace, :undef => :replace, :replace => '?')
+      url = url.gsub(/\s/, '')
+      url = url.gsub(ANCHOR_REGEX, '')
+      url = unescape(url)
+      m = URL_REGEX.match(url.downcase)
+      if m
+        protocol, host, port, dir, query = m[:protocol], m[:host], m[:port], m[:dir], m[:query]
+        protocol = 'http://' if protocol.nil? or protocol == '/'
+        host = host.sub(/\A\.*/,'').sub(/\.\z/, '') if host
+        dir = dir.sub(/\A\/*/, '').gsub(/\/+/, '/').gsub(/\/\.\//, '/') if dir
+        url = protocol << host.to_s << port.to_s << '/' << dir.to_s << query.to_s
+      end
+      
+      url
+    end
+    
+    def unescape url
+      unescape = URI.unescape(url)
+      while unescape != url
+        url = unescape
+        unescape = URI.unescape(url)
+      end
+      URI.escape(unescape)
+    end
+    
+  end
+end

data/lib/google/url_scramble.rb

@@ -0,0 +1,54 @@
+module Google
+  module UrlScramble
+    extend self
+    
+    # url is canonicalized url
+    def gen url
+      m = Google::UrlCanonicalizer::URL_REGEX.match(url)
+      return [[], []] if m.nil?
+      protocol, host, port, dir, query = m[:protocol], m[:host], m[:port], m[:dir], m[:query]
+      return [[], []] if host.nil?
+
+      urls = []
+      hosts, paths = [], []
+
+      hosts << host
+      if /[^\d.]/ =~ host
+        host_segments = host.split('.')
+        host_segments = host_segments[-6..-1] if host_segments.size >= 6
+        h = host_segments.shift
+        while !h.nil? && host_segments.size > 1 && hosts.size <= 5
+          hosts << host_segments.join('.')
+          h = host_segments.shift
+        end
+        host_keys = hosts.select {|x| x.count(".") == 2 or x.count(".") == 1}.map{|x| x+'/'}
+      else
+        host_keys = hosts.map{|x| x+'/'}
+      end
+      
+      dir = dir.to_s.sub(/\A\//, '') # remove the leading slash
+      paths << ('/' << dir)
+      paths << ('/' << dir << query.to_s) unless query.blank?
+      path_segments = dir.split('/')
+      paths << '/'
+      count = 0; tmp_path = ''
+      while !(p = path_segments.shift).nil? && count <= 3
+        tmp_path += ('/' << p)
+        paths << tmp_path
+        count += 1
+      end
+      paths.uniq!
+      
+      (urls = []).tap do
+        hosts.each do |h|
+          paths.each do |p|
+            urls << h + p
+          end
+        end
+      end
+      
+      [host_keys, urls]
+    end
+    
+  end
+end

data/lib/google_safe_browsing_plugin.rb

@@ -0,0 +1,29 @@
+require 'active_support/all'
+
+module Google
+  def self.table_name_prefix
+    'google_'
+  end
+  
+  if defined?(Rails)
+    config_file = File.expand_path('./config/google_safe_browsing.yml', Rails.root)
+    CONFIG ||= YAML.load_file(config_file)[Rails.env] if File.exists?(config_file)
+  end
+
+  autoload :SafeBrowsingClient, 'google/safe_browsing_client'
+  autoload :SafeBrowsingParser, 'google/safe_browsing_parser'
+  autoload :SafeBrowsingUpdateHelper, 'google/safe_browsing_update_helper'  
+  autoload :ShaUtil, 'google/sha_util'
+  autoload :UrlCanonicalizer, 'google/url_canonicalizer'
+  autoload :UrlScramble, 'google/url_scramble'
+end
+
+module Faraday
+  class Response
+    autoload :SafeBrowsingUpdateParser, 'faraday/response/safe_browsing_update_parser'
+  end
+end
+
+if defined?(Rake)
+  require 'safe_browsing_task'
+end

data/lib/rails/generators/google/config/config_generator.rb

@@ -0,0 +1,16 @@
+# encoding: utf-8
+
+module Google
+  module Generators
+    class ConfigGenerator < Rails::Generators::Base
+      desc 'Creates a Google Safe Browsing plugin configuration file config/google_safe_browsing.yml'
+
+      source_root File.expand_path('../templates', __FILE__)
+
+      def copy_config_file
+        copy_file 'google_safe_browsing.yml', "config/google_safe_browsing.yml"
+      end
+
+    end
+  end
+end

data/lib/rails/generators/google/config/templates/google_safe_browsing.yml

@@ -0,0 +1,16 @@
+default: &default
+  api_key: your_google_api_key
+  app_ver: 1.5.2
+  p_ver: 2.2
+  full_length_hash_expires: 86400 # 1 day
+  full_length_hash_backoff_delay: 300 # seconds
+  full_length_hash_backoff_delay_max: 7200 # seconds
+      
+development:
+  <<: *default
+
+test:
+  <<: *default
+ 
+production:
+  <<: *default

data/lib/rails/generators/google/helper/helper_generator.rb

@@ -0,0 +1,16 @@
+# encoding: utf-8
+
+module Google
+  module Generators
+    class HelperGenerator < Rails::Generators::Base
+      desc 'Creates a Google Safe Browsing plugin helper file app/helpers/google/safe_browsing_helper.rb'
+
+      source_root File.expand_path('../templates', __FILE__)
+
+      def copy_helper_file
+        copy_file 'safe_browsing_helper.rb', "app/helpers/google/safe_browsing_helper.rb"
+      end
+
+    end
+  end
+end

data/lib/rails/generators/google/helper/templates/safe_browsing_helper.rb

@@ -0,0 +1,168 @@
+module Google
+  module SafeBrowsingHelper
+    extend self
+  
+    def save_full_hash_requests prefixes, state, requested_at = Time.now
+      prefixes.each do |pre|
+        obj = SafeBrowsingFullHashRequest.where(prefix: pre).first
+        if obj.nil?
+          obj = SafeBrowsingFullHashRequest.create(prefix: pre, requested_at: requested_at, state: state)
+        else
+          obj.update_attributes(requested_at: requested_at, state: state)
+        end
+      end
+    end
+    
+  
+    #
+    # hash_prefix_arr: ['5b3583c0', 'b3e357a6']
+    #
+    def get_full_hash_list hash_prefix_arr
+      ret_full_hashes = []
+      api =  SafeBrowsingClient.new
+
+      return ret_full_hashes if hash_prefix_arr.empty?
+      
+      begin
+        full_hash_objs = api.full_hash(hash_prefix_arr) || {}
+        save_full_hash_requests(hash_prefix_arr, SafeBrowsingFullHashRequest::COMPLETED)
+      rescue Google::Error::NoContent => e
+        Rails.logger.warn "NoContent Error for hash prefixes [#{hash_prefix_arr.join(', ')}]"
+        save_full_hash_requests(hash_prefix_arr, SafeBrowsingFullHashRequest::COMPLETED)
+        return ret_full_hashes
+      rescue Exception => e
+        Rails.logger.error "Error backtrace #{e.backtrace.join(%Q(\n))}"
+        Rails.logger.warn "Error (#{e.inspect}) for hash prefixes [#{hash_prefix_arr.join(', ')}], continue..."
+        save_full_hash_requests(hash_prefix_arr, e.message)
+        return ret_full_hashes
+      end
+        
+      full_hash_objs.keys.each do |list|
+        full_hash_objs[list].keys.each do |add_chunk_num|
+          full_hash_objs[list][add_chunk_num].each do |full_hash|
+            unless (list_obj = SafeBrowsingList.list_by_name(list)).nil?
+              Rails.logger.info "Updating full hash data with #{list}:#{add_chunk_num}:#{full_hash}"
+              local = SafeBrowsingFullHash.where(value: full_hash).first
+
+              if local
+                local.touch
+              else
+                local = SafeBrowsingFullHash.create(value: full_hash, add_chunk_num: add_chunk_num, 
+                                        google_safe_browsing_list_id: list_obj.id)
+              end
+              
+              ret_full_hashes << full_hash
+            end
+          end
+        end
+      end
+      
+      ret_full_hashes
+    end
+    
+    def can_request_full_hash? hash_prefix
+      now = Time.now
+      request = Google::SafeBrowsingFullHashRequest.where(prefix: hash_prefix).first
+      if request.nil?
+        true
+      else
+        if request.state == SafeBrowsingFullHashRequest::COMPLETED
+          if request.requested_at < now - Google::CONFIG['full_length_hash_expires']
+            true
+          else
+            Rails.logger.warn "Full hash [#{hash_prefix}] requested successfully recently, skip this time."
+            false
+          end
+        else
+          attempts = request.attempts.nil?? 1 : request.attempts
+          max_delay = Google::CONFIG['full_length_hash_backoff_delay_max']
+          delay = attempts * Google::CONFIG['full_length_hash_backoff_delay']
+          delay = max_delay if delay > max_delay
+          if request.requested_at > now - delay
+            Rails.logger.warn "Full hash [#{hash_prefix}] request in backoff mode. Wait time is #{delay} seconds"
+            false
+          else
+            true
+          end
+        end
+      end
+    end
+    
+    def lookup_url url
+      canon_url = Google::UrlCanonicalizer.apply(url)
+      hosts_set, urls_set = Google::UrlScramble.gen(canon_url)
+      full_hits = []
+      
+      unless hosts_set.empty?
+        host_shas = hosts_set.map {|x| ShaUtil.sha256_hex(x, 8)}
+        host_hits = find_host_key_hits(host_shas).map(&:host_key)
+        
+        if host_hits.empty?
+          Rails.logger.info "No host key prefix found. Return."
+          return hits_to_category(url, full_hits)
+        else
+          url_shas = urls_set.map {|x| ShaUtil.sha256_hex(x, 8)}
+          prefix_hits = find_prefix_key_hits(host_shas, url_shas).map(&:prefix)
+          
+          candidate_prefixes = (host_hits + prefix_hits).uniq
+          full_hash_expressions = urls_set.map {|x| ShaUtil.sha256_hex(x)}.select {|x| candidate_prefixes.include?(x.first(8))}
+          full_hits = full_hash_cache_hits(full_hash_expressions)
+
+          if full_hits.empty?
+            warm_prefixes = candidate_prefixes.select {|x| !can_request_full_hash?(x)}
+            candidate_prefixes -= warm_prefixes
+            unless candidate_prefixes.empty?
+              Rails.logger.info "Asking Google for full length hash #{candidate_prefixes}"
+              get_full_hash_list(candidate_prefixes) 
+              full_hits = full_hash_cache_hits(full_hash_expressions)
+            end
+          end
+        end
+      end
+
+      if full_hits.empty?
+        Rails.logger.info "Return no hits..."
+      else
+        Rails.logger.info "Return hits #{full_hits.inspect}..."
+      end
+
+      hits_to_category(url, full_hits)
+    end
+    
+    def hits_to_category url, full_hits
+      (ret = {}).tap do
+        ret[url] ||= []
+        full_hits.each do |hit|
+          ret[url] << hit.list.name
+        end
+      end
+    end
+    
+    def find_host_key_hits prefixes
+      ret = []
+      SafeBrowsingShavar.add_host_keys(prefixes).each do |add|
+        if SafeBrowsingShavar.find_subs_for_add(add.chunk_num, add.host_key, add.prefix).empty?
+          ret << add
+        end
+      end
+      ret
+    end
+    
+    def find_prefix_key_hits host_keys, prefixes
+      ret = []
+      SafeBrowsingShavar.add_host_prefixes(host_keys, prefixes).each do |add|
+        if SafeBrowsingShavar.find_subs_for_add(add.chunk_num, add.host_key, add.prefix).empty?
+          ret << add
+        end
+      end
+      ret
+    end
+    
+    def full_hash_cache_hits full_hashes
+      SafeBrowsingFullHash.includes(:list).where(value: full_hashes)
+                          .where('updated_at > ?', Time.now - Google::CONFIG['full_length_hash_expires'])
+    end
+        
+    
+  end
+end