google_safe_browsing 0.6.3 → 0.6.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 5f4218a52f77b7962936b3d80ceac90edce5ee52
-  data.tar.gz: a46bc606764928cc45cb99fcc21f0170e2c9b32a
+  metadata.gz: aaec9b161ce5eb3c820b5bd759c0a8b7c98844d1
+  data.tar.gz: 37dfc9d45dabf167e75cd7ec9dd8e5e4136bc42d
 SHA512:
-  metadata.gz: c13d1c2422be06b106dcb372cffcfbcc6efa4994dba0befa51556d5b4612c8deab615d799c05b8690ac5914e325b7772f26c33fde82a0d7e6e6911d3428eba17
-  data.tar.gz: bc86c27199206072d26103399c48b6833b8aebbc2ecabf9144ef8c832e0b02a00019ff6b88e36d11a527096d2fc681e2b2e8b92a548131744f187b81826ec24d
+  metadata.gz: 619979715cecc6b6e782e15a080beed3b4a5919dbe6c0c64aa600427523e6111a1e3c5b851c3975032cbeb0a9911a9c74ce38bc4eeb00dd13d298df1b3f30d48
+  data.tar.gz: 064d1395118e4e768e290fbe9b9d15ccd99b9d96e2d1e7af2f5a601c721d17f22a5a2c809e9110dd4c2d602b759b567fc56fb1497c875e3e7d024a6a7bd65c79

data/README.md

@@ -98,7 +98,7 @@ not on Google's list.
 ## Contributing
 
 We've already had some [great
-contributers](https://github.com/mobiledefense/google_safe_browsing/graphs/contributors).
+contributors](https://github.com/mobiledefense/google_safe_browsing/graphs/contributors).
 If you'd like to join us, we'd love to have you. When contributing please
 
 1. [Fork](https://github.com/mobiledefense/google_safe_browsing/fork) the repo

data/lib/google_safe_browsing.rb

@@ -46,33 +46,41 @@ module GoogleSafeBrowsing
     end
   end
 
-  # Returns of initializes the Module configuration
-  def self.config
-    @@config ||= Config.new
-  end
+  class << self
+    attr_accessor :logger
 
-  # Allows for setting config values via a block
-  def self.configure
-    yield self.config
-  end
+    def logger
+      @logger ||= Logger.new
+    end
 
-  # Adds the Rescheduler job to Resque
-  def self.kick_off
-    Resque.enqueue(Rescheduler)
-  end
+    # Returns of initializes the Module configuration
+    def config
+      @@config ||= Config.new
+    end
+
+    # Allows for setting config values via a block
+    def configure
+      yield config
+    end
+
+    # Adds the Rescheduler job to Resque
+    def kick_off
+      Resque.enqueue(Rescheduler)
+    end
 
-  # Converts the official Google list name into the name to return
-  #
-  # @param (String) list the 'official' list name
-  # @return (String) the friendly list name
-  def self.friendly_list_name(list)
-    case list
-    when 'goog-malware-shavar'
-      'malware'
-    when 'googpub-phish-shavar'
-      'phishing'
-    else
-      nil
+    # Converts the official Google list name into the name to return
+    #
+    # @param (String) list the 'official' list name
+    # @return (String) the friendly list name
+    def friendly_list_name(list)
+      case list
+      when 'goog-malware-shavar'
+        'malware'
+      when 'googpub-phish-shavar'
+        'phishing'
+      else
+        nil
+      end
     end
   end
 end

data/lib/google_safe_browsing/api_v2.rb

@@ -13,7 +13,7 @@ module GoogleSafeBrowsing
 
       to_do_array[:lists].each do |list|
         to_do_array[:data_urls][list].each do |url|
-          puts "#{list} - #{url}\n"
+          GoogleSafeBrowsing.logger.info "#{list} - #{url}\n"
           ResponseHelper.receive_data('http://' + url, list)
         end
       end
@@ -29,26 +29,26 @@ module GoogleSafeBrowsing
       return nil if urls.empty?
 
       hashes = HashHelper.urls_to_hashes(urls)
-      raw_hash_array = hashes.collect{ |h| h.to_s }
+      raw_hash_array = hashes.map { |h| h.to_s }
 
-      if full = FullHash.where(:full_hash => raw_hash_array).first
-        return GoogleSafeBrowsing.friendly_list_name(full.list)
-      end
+      full = FullHash.where(full_hash: raw_hash_array).first
+      return GoogleSafeBrowsing.friendly_list_name(full.list) if full
 
-      hits =  AddShavar.where(:prefix => hashes.map{|h| h.prefix}).collect{ |s| [ s.list, s.prefix ] }
-      safes = SubShavar.where(:prefix => hashes.map{|h| h.prefix}).collect{ |s| [ s.list, s.prefix ] }
+      hits =  AddShavar.where(prefix: hashes.map { |h| h.prefix }).map { |s| [s.list, s.prefix] }
+      safes = SubShavar.where(prefix: hashes.map { |h| h.prefix }).map { |s| [s.list, s.prefix] }
 
       reals = hits - safes
 
       if reals.any?
-        full_hashes = HttpHelper.request_full_hashes(reals.collect{|r| r[1] })
+        full_hashes = HttpHelper.request_full_hashes(reals.map { |r| r[1] })
 
         # save hashes first
         # cannot return early because all FullHashes need to be saved
         hit_list = nil
         full_hashes.each do |hash|
-          FullHash.create!(:list => hash[:list], :add_chunk_number => hash[:add_chunk_num],
-                                       :full_hash => hash[:full_hash])
+          FullHash.create!(list: hash[:list],
+                           add_chunk_number: hash[:add_chunk_num],
+                           full_hash: hash[:full_hash])
 
           hit_list = hash[:list] if raw_hash_array.include?(hash[:full_hash])
         end
@@ -59,16 +59,19 @@ module GoogleSafeBrowsing
 
     # Can be used to force a delay into a script running updates
     #
-    # @param (Integer) delay_seconds the number of seconds to delay, should be the return value of {update}
+    # @param (Integer) delay_seconds the number of seconds to delay, should be
+    # the return value of {update}
     def self.delay(delay_seconds)
-      puts "Google told us to wait for #{delay_seconds} seconds"
-      puts "We will wait...."
+      GoogleSafeBrowsing.logger.info \
+        "Google told us to wait for #{delay_seconds} seconds"
+      GoogleSafeBrowsing.logger.info 'We will wait....'
       start_time = Time.now
-      while(start_time + delay_seconds > Time.now)
-          puts "#{(delay_seconds - (Time.now - start_time)).to_i}..."
+      until start_time + delay_seconds <= Time.now
+          GoogleSafeBrowsing.logger.info \
+            "#{(delay_seconds - (Time.now - start_time)).to_i}..."
           sleep(10)
       end
-      puts "Thank you for being patient"
+      GoogleSafeBrowsing.logger.info 'Thank you for being patient'
     end
   end
 end

data/lib/google_safe_browsing/binary_helper.rb

@@ -1,7 +1,6 @@
 module GoogleSafeBrowsing
   # Helper methods for working with binary encoded data from Forwarding URLs
   class BinaryHelper
-
     # Reads `counter` byes from byte iterator `iter` and returns the hex string represnetation
     #
     # @param [ByteIterator] iter byte iterator already at correct position
@@ -19,7 +18,6 @@ module GoogleSafeBrowsing
       string.unpack('H8')[0]
     end
 
-
     # Read `count` bytes from `iter` without unpacking the result
     #
     # @param (see read_bytes_as_hex)
@@ -30,9 +28,6 @@ module GoogleSafeBrowsing
       ret = ''
       count.to_i.times { ret << iter.next }
       ret
-   #rescue
-   #  puts "Tried to read past chunk iterator++++++++++++++++++++++++++++++++++++++++++++++++++++++++"
-   #  return nil
     end
 
     # Returns the first four bytes of `string` as hexidecimal; for host key
@@ -66,6 +61,5 @@ module GoogleSafeBrowsing
     def self.hex_to_bin(hex)
       [hex].pack('H*')
     end
-
   end
 end

data/lib/google_safe_browsing/canonicalize.rb

@@ -5,7 +5,6 @@ require File.dirname(__FILE__) + '/top_level_domain.rb'
 module GoogleSafeBrowsing
   # Helpers to Canonicalize urls and generate url permutations for lookups
   class Canonicalize
-
     PROTOCOL_DELIMITER = '://'
     DEFAULT_PROTOCOL = 'http'
 
@@ -15,14 +14,15 @@ module GoogleSafeBrowsing
     # @return (String) canonicalized url string
     def self.url(raw_url)
       raw_url = raw_url.to_s
-      
-      # Change encoding from UTF-8 to ASCII-8BIT to avoid InvalidByteSequenceError
-      raw_url = raw_url.force_encoding("ASCII-8BIT")
 
-      #remove tabs, carriage returns and line feeds
-      raw_url.gsub!("\t",'')
-      raw_url.gsub!("\r",'')
-      raw_url.gsub!("\n",'')
+      # Change encoding from UTF-8 to ASCII-8BIT to avoid
+      # InvalidByteSequenceError
+      raw_url = raw_url.force_encoding('ASCII-8BIT')
+
+      # remove tabs, carriage returns and line feeds
+      raw_url.gsub!("\t", '')
+      raw_url.gsub!("\r", '')
+      raw_url.gsub!("\n", '')
 
       cann = raw_url.clone
       cann.gsub!(/\A\s+|\s+\Z/, '')
@@ -35,10 +35,10 @@ module GoogleSafeBrowsing
       # remove leading PROTOCOL
       cann = remove_protocol(cann)
 
-      #split into host and path components
+      # split into host and path components
       splits = split_host_path(cann)
 
-      cann = fix_host( splits[:host] ) + '/' + fix_path( splits[:path] )
+      cann = fix_host(splits[:host]) + '/' + fix_path(splits[:path])
 
       # add leading protocol
       @protocol ||= DEFAULT_PROTOCOL
@@ -53,7 +53,7 @@ module GoogleSafeBrowsing
     # @return (Array) array of cannonicalized url permutation strings
     def self.urls_for_lookup(lookup_url)
       lookup_url = url(lookup_url)
-      #return empty array if url returns nil; for invalid url
+      # return empty array if url returns nil; for invalid url
       return [] if lookup_url.blank?
 
       lookup_url = remove_protocol(lookup_url)
@@ -62,12 +62,12 @@ module GoogleSafeBrowsing
 
       host_string = strip_username_password_and_port_from_host(splits[:host])
 
-      #return empty array unless host_string has at least one period
+      # return empty array unless host_string has at least one period
       return [] unless host_string.include?('.')
 
       host_strings = [host_string]
       host = TopLevelDomain.split_from_host(host_string).last(5)
-      ( host.length - 1 ).times do 
+      (host.length - 1).times do
         host_strings << host.join('.')
         host.shift
       end
@@ -78,52 +78,43 @@ module GoogleSafeBrowsing
       cart_prod(host_strings, path_strings)
     end
 
-    # private
-
     # Generates the path permutations from the raw path string
     #
     # @param (String) raw_path path split from the full url string
     # @return (Array) array of path permutation strings
     def self.generate_path_strings(raw_path)
-      return [ '/', '' ] if raw_path == ''
+      return ['/', ''] if raw_path == ''
 
       path_split = raw_path.split('?')
       path = path_split[0] || ''
       params = path_split[1] || ''
 
-
       path_components = path.split('/').first(3)
-      path_strings = [ '/' ]
+      path_strings = ['/']
       path_components.length.times do
         path_strings << '/' + path_components.join('/')
         path_components.pop
       end
 
       path_strings.map! do |p|
-        unless p.index('.')
-          p + '/'
-        else
+        if p.index('.')
           p
+        else
+          p + '/'
         end
       end
-      path_strings.map!{ |p| p.to_s.gsub!(/\/+/, '/') }
+      path_strings.map! { |p| p.to_s.gsub!(/\/+/, '/') }
       path_strings.compact!
       path_strings.uniq!
 
-      unless params.blank?
-        path_strings | path_strings.map do |p|
-          if p[-1] == '/'
-            p
-          else
-            "#{p}?#{params}"
-          end
-        end
-      else
-        return path_strings
+      return path_strings if params.blank?
+      path_strings | path_strings.map do |p|
+        p[-1] == '/' ?  p : "#{p}?#{params}"
       end
     end
 
-    # Returns the cartesian product of two arrays by concatination of the string representation of the elements
+    # Returns the cartesian product of two arrays by concatination of the
+    # string representation of the elements
     #
     # @param (Array) a_one array of strings
     # @param (Array) a_two array of strings
@@ -141,24 +132,25 @@ module GoogleSafeBrowsing
     # Takes the canonicalized url and splits the host and the path apart
     #
     # @param (String) cann canonicalized url string
-    # @return (Hash) !{ :host => host_part, :path => path_part }
+    # @return (Hash) !{ host: host_part, path: path_part }
     def self.split_host_path(cann)
-      ret= { :host => cann, :path => '' }
+      ret = { host: cann, path: '' }
       split_point = cann.index('/')
       if split_point
-        ret[:host] = cann[0..split_point-1]
-        ret[:path] = cann[split_point+1..-1]
+        ret[:host] = cann[0..split_point - 1]
+        ret[:path] = cann[(split_point + 1)..-1]
       end
 
       ret
     end
 
-    # Strips the fragment portion of the url string (the last '#' and everything after)
+    # Strips the fragment portion of the url string (the last '#' and
+    # everything after)
     #
     # @param (String) string url
     # @return (String) parameter with the fragment removed
     def self.remove_fragment(string)
-      string = string[0..string.index('#')-1] if string.index('#')
+      string = string[0..(string.index('#') - 1)] if string.index('#')
       string
     end
 
@@ -167,9 +159,9 @@ module GoogleSafeBrowsing
     # @param (String) url url string
     # @return (String) fully unescaped url string
     def self.recursively_unescape(url)
-      compare_url = url.clone 
+      compare_url = url.clone
       url = URI.unescape(url)
-      while(compare_url != url)
+      until compare_url == url
         compare_url = url.clone
         url = URI.unescape(url)
       end
@@ -181,7 +173,6 @@ module GoogleSafeBrowsing
     # @param (String) host host string
     # @return (String) standardized host string
     def self.fix_host(host)
-      #puts "In Host: #{host}"
       # remove leading and trailing dots, multiple dots to one
       host.gsub!(/\A\.+|\.+\Z/, '')
       host.gsub!(/\.+/, '.')
@@ -194,7 +185,7 @@ module GoogleSafeBrowsing
         host_splits[:host] = IP::V4.new(host.to_i).to_addr
       elsif host_splits[:host] =~ /\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}/
         begin
-          host_splits[:host] = IP.new(host).to_addr 
+          host_splits[:host] = IP.new(host).to_addr
         rescue ArgumentError
         end
       end
@@ -210,17 +201,15 @@ module GoogleSafeBrowsing
     # @param (String) path path string
     # @return (String) standardized path string
     def self.fix_path(path)
-      #puts "In Path: #{path}"
-
-      #remove leading slash
+      # remove leading slash
       path = path[1..-1] if path[0..0] == '/'
 
-      preserve_trailing_slash = ( path[-1..-1] == '/' )
+      preserve_trailing_slash = (path[-1..-1] == '/')
 
       if path.index('?')
         first_ques = path.index('?')
         params = path[first_ques..-1]
-        path = path[0..first_ques-1]
+        path = path[0..(first_ques - 1)]
       end
 
       # remove multiple '/'
@@ -247,7 +236,7 @@ module GoogleSafeBrowsing
       url = URI.escape url
 
       # unescape carat, may need other optionally escapeable chars
-      url.gsub!('%5E','^')
+      url.gsub!('%5E', '^')
 
       url
     end
@@ -259,7 +248,7 @@ module GoogleSafeBrowsing
     def self.remove_protocol(cann)
       if cann.index(PROTOCOL_DELIMITER)
         delimiting_index = cann.index(PROTOCOL_DELIMITER)
-        @protocol = cann[0..delimiting_index-1]
+        @protocol = cann[0..(delimiting_index - 1)]
         protocol_end_index = delimiting_index + PROTOCOL_DELIMITER.length
         cann = cann[protocol_end_index..-1]
       end

data/lib/google_safe_browsing/google_safe_browsing_railtie.rb

@@ -15,5 +15,9 @@ module GoogleSafeBrowsing
         config.api_key = app.config.google_safe_browsing[:api_key]
       end
     end
+
+    initializer 'Rails logger' do
+      GoogleSafeBrowsing.logger = Rails.logger
+    end
   end
 end

data/lib/google_safe_browsing/http_helper.rb

@@ -57,6 +57,8 @@ module GoogleSafeBrowsing
           GoogleSafeBrowsing.config.client_key = key_value
         when 'wrappedkey'
           GoogleSafeBrowsing.config.wrapped_key = key_value
+        else
+          GoogleSafeBrwosing::Logger.warn "Unknown MAC key: #{key_name}"
         end
       end
     end

data/lib/google_safe_browsing/rescheduler.rb

@@ -3,11 +3,11 @@ module GoogleSafeBrowsing
     @queue = :google_safe_browsing
 
     def self.perform
-      puts "Running Update"
+      GoogleSafeBrowsing.logger.info "Running Update"
       delay = APIv2.update
-      puts "Scheduling new update in #{delay} seconds"
+      GoogleSafeBrowsing.logger.info "Scheduling new update in #{delay} seconds"
       Resque.enqueue_in(delay.seconds, Rescheduler)
-      puts "Update scheduled"
+      GoogleSafeBrowsing.logger.info "Update scheduled"
     end
 
   end

data/lib/google_safe_browsing/response_helper.rb

@@ -49,6 +49,9 @@ module GoogleSafeBrowsing
           # we no longer have to report that we received these chunks
           SubShavar.delete_chunks_from_list(current_list,
                                             ChunkList.new(vals[1]))
+        else
+          GoogleSafeBrowsing.logger.warn \
+            "Unknown data response directive: #{vals[0]}"
         end
       end
 
@@ -75,9 +78,11 @@ module GoogleSafeBrowsing
           chunk  = f.read(line_actions[:chunk_length])
           # f iterator is now set for next chunk
 
-          add_attrs = { :chunk_number => line_actions[:chunk_number],
-                    :list => list, :prefix => nil, :host_key => nil
-                  }
+          add_attrs = { chunk_number: line_actions[:chunk_number],
+                        list: list,
+                        prefix: nil,
+                        host_key: nil
+                      }
 
           case line_actions[:action]
           when 'a'
@@ -102,11 +107,14 @@ module GoogleSafeBrowsing
                   counter += 1
                 end
               rescue StopIteration
-                puts "Added #{counter} host_keys for add chunk number #{line_actions[:chunk_number]}"
+                GoogleSafeBrowsing.logger.info <<-LOG.gsub(/\n\s*/, '')
+                  Added #{counter} host_keys for add chunk number
+                  #{line_actions[:chunk_number]}
+                  LOG
               end
             end
           when 's'
-            sub_attrs = add_attrs.merge({ :add_chunk_number => nil })
+            sub_attrs = add_attrs.merge({ add_chunk_number: nil })
             if line_actions[:chunk_length] == 0
               record_sub_shavar_to_insert(sub_attrs)
             else
@@ -130,11 +138,16 @@ module GoogleSafeBrowsing
                   counter += 1
                 end
               rescue StopIteration
-                puts "Added #{counter} host_keys for sub chunk number #{line_actions[:chunk_number]}"
+                GoogleSafeBrowsing.logger.info <<-LOG.gsub(/\n\s*/, '')
+                  Added #{counter} host_keys for sub chunk number
+                  #{line_actions[:chunk_number]}
+                LOG
               end
             end
           else
-            puts "neither a nor s ======================================================="
+            GoogleSafeBrowsing.logger.info <<-LOG.gsub(/\n\s*/, '')
+              neither a nor s =================================================
+            LOG
           end
 
         end
@@ -165,16 +178,16 @@ module GoogleSafeBrowsing
 
     def self.record_add_shavar_to_insert(h)
       @add_shavar_values ||= []
-      values = [ h[:prefix], h[:host_key], h[:chunk_number], h[:list] ]
+      values = [h[:prefix], h[:host_key], h[:chunk_number], h[:list]]
       @add_shavar_values << "(#{escape_and_join values})"
     end
     def self.record_sub_shavar_to_insert(h)
       @sub_shavar_values ||= []
-      values = [ h[:prefix],
-                 h[:host_key],
-                 h[:add_chunk_number],
-                 h[:chunk_number],
-                 h[:list] ]
+      values = [h[:prefix],
+                h[:host_key],
+                h[:add_chunk_number],
+                h[:chunk_number],
+                h[:list]]
       @sub_shavar_values << "(#{escape_and_join values})"
     end
 
@@ -182,10 +195,10 @@ module GoogleSafeBrowsing
       split_line = line.split(':')
 
       ret = {}
-      ret[ :action ]        = split_line[0]
-      ret[ :chunk_number ]  = split_line[1].to_i
-      ret[ :hash_length ]   = split_line[2].to_i
-      ret[ :chunk_length ]  = split_line[3].to_i
+      ret[:action]        = split_line[0]
+      ret[:chunk_number]  = split_line[1].to_i
+      ret[:hash_length]   = split_line[2].to_i
+      ret[:chunk_length]  = split_line[3].to_i
       ret
     end
 
@@ -196,7 +209,7 @@ module GoogleSafeBrowsing
     end
 
     def self.pop_and_join(records)
-      records.pop(10000).join(', ')
+      records.pop(10_000).join(', ')
     end
   end
 end

data/lib/google_safe_browsing/top_level_domain.rb

@@ -38,13 +38,12 @@ module GoogleSafeBrowsing
       components
     end
 
-
-    private 
+    private
 
     def self.parse_tld_to_hash
       hash = Hash.new(nil)
-      f = File.open(File.dirname(__FILE__) + '/effective_tld_names.dat.txt', 'r')
-      while(line = f.gets)
+      file_name = File.dirname(__FILE__) + '/effective_tld_names.dat.txt'
+      File.readlines(file_name, 'r').each do |line|
         hash[line.chomp] = true unless line[0..1] == '//'
       end
       hash

data/lib/google_safe_browsing/version.rb

@@ -1,3 +1,3 @@
 module GoogleSafeBrowsing
-  VERSION = "0.6.3"
+  VERSION = "0.6.4"
 end

metadata

@@ -1,29 +1,29 @@
 --- !ruby/object:Gem::Specification
 name: google_safe_browsing
 version: !ruby/object:Gem::Version
-  version: 0.6.3
+  version: 0.6.4
 platform: ruby
 authors:
 - Chris Marshall
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-11-03 00:00:00.000000000 Z
+date: 2015-07-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rails
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '3.2'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '3.2'
 - !ruby/object:Gem::Dependency
   name: ruby-ip
   requirement: !ruby/object:Gem::Requirement
@@ -157,7 +157,8 @@ files:
 - lib/google_safe_browsing/version.rb
 - lib/tasks/google_safe_browsing_tasks.rake
 homepage: https://github.com/mobiledefense/google_safe_browsing
-licenses: []
+licenses:
+- Apache 2.0
 metadata: {}
 post_install_message: 
 rdoc_options: []
@@ -175,7 +176,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.2.2
+rubygems_version: 2.4.5
 signing_key: 
 specification_version: 4
 summary: Rails 3 plugin for Google's Safe Browsing API v2