google_safe_browsing 0.3.0 → 0.3.1

data/README.mkd

@@ -1,15 +1,15 @@
-# Google Safe Browsing Rails 3 Plugin
+# Google Safe Browsing Gem
 
-This gem allows easy Google Safe Browsing integration
-with Rails 3 apps.
+This gem allows easy Google Safe Browsing APIv2 usage with optional integration
+into Rails 3 apps.
 
 It includes:
 
 * a migration generator for database schema
-* method to update your lists
+* method to update hash database
 * method to lookup a url
 * rake tasks to update hash database
-* Autonomous updating via Resque and Resque Scheduler
+* Autonomous updating via Resque and Resque Scheduler (optional)
 
 ----------------------
 
@@ -96,6 +96,4 @@ https://github.com/codelux/malware_api
 
 ------------------
 
-Thank you for using my gem! Please report any bugs or issues. Contributions are also always welcome!
-
--- Chris Marshall
+Thank you for using this gem! Please report any bugs or issues.

data/lib/google_safe_browsing.rb

@@ -20,6 +20,8 @@ require File.dirname(__FILE__) + '/google_safe_browsing/full_hash'
 require File.dirname(__FILE__) + '/google_safe_browsing/rescheduler'
 
 module GoogleSafeBrowsing
+
+  # Handles the configuration values for the module
   class Config
     attr_accessor :client, :app_ver, :p_ver, :host, :current_lists, :api_key
 
@@ -32,19 +34,26 @@ module GoogleSafeBrowsing
     end
   end
 
+  # Returns of initializes the Module configuration
   def self.config
     @@config ||= Config.new
   end
 
+  # Allows for setting config values via a block
   def self.configure
     yield self.config
   end
 
+  # Adds the Rescheduler job to Resque
   def self.kick_off
     Resque.enqueue(Rescheduler)
   end
 
 
+  # Converts the official Google list name into the name to return
+  #
+  # @param (String) list the 'official' list name
+  # @return (String) the friendly list name
   def self.friendly_list_name(list)
     case list
     when 'goog-malware-shavar'

data/lib/google_safe_browsing/api_v2.rb

@@ -1,5 +1,9 @@
 module GoogleSafeBrowsing
+  # Main Interface for Module
   class APIv2
+    # Completes an update
+    #
+    # @return (Integer) the number of seconds before this method should be called again
     def self.update
       data_response = HttpHelper.get_data
 
@@ -14,8 +18,13 @@ module GoogleSafeBrowsing
       to_do_array[:delay_seconds]
     end
 
+    # Performs a lookup of the given url
+    #
+    # @param (String) url a url string to be looked up
+    # @return (String, nil) the friendly list name if found, or `nil` 
     def self.lookup(url)
       urls = Canonicalize.urls_for_lookup(url)
+      return '' if urls.empty?
 
       hashes = HashHelper.urls_to_hashes(urls)
       raw_hash_array = hashes.collect{ |h| h.to_s }
@@ -46,6 +55,9 @@ module GoogleSafeBrowsing
       nil
     end
 
+    # Can be used to force a delay into a script running updates
+    #
+    # @param (Integer) delay_seconds the number of seconds to delay, should be the return value of {update}
     def self.delay(delay_seconds)
       puts "Google told us to wait for #{delay_seconds} seconds"
       puts "We will wait...."

data/lib/google_safe_browsing/binary_helper.rb

@@ -1,15 +1,29 @@
 module GoogleSafeBrowsing
+  # Helper methods for working with binary encoded data from Forwarding URLs
   class BinaryHelper
 
+    # Reads `counter` byes from byte iterator `iter` and returns the hex string represnetation
+    #
+    # @param [ByteIterator] iter byte iterator already at correct position
+    # @param [Integer] count number of bytes to read
+    # @return [String] hexidecimal string
     def self.read_bytes_as_hex(iter, count)
       read_bytes_from(iter, count).unpack("H#{count * 2}")[0]
     end
 
+    # Returns the first four bytes of `string` as hexidecimal
+    #
+    # @param [String] string to unpack the first four bytes as hex
+    # @return (see read_bytes_as_hex)
     def self.four_as_hex(string)
       string.unpack('H8')[0]
     end
 
 
+    # Read `count` bytes from `iter` without unpacking the result
+    #
+    # @param (see read_bytes_as_hex)
+    # @return (String) not unpacked string from `iter`
     def self.read_bytes_from(iter, count)
       ret = ''
       count.to_i.times { ret << iter.next }
@@ -19,19 +33,34 @@ module GoogleSafeBrowsing
    #  return nil
     end
 
+    # Returns the first four bytes of `string` as hexidecimal; for host key
+    # @param (String) bin string to unpack
+    # @return (String) unpacked string
     def self.unpack_host_key(bin)
       bin.unpack('H8')[0]
     end
 
+    # Unpack string as an unsigned integer; for count
+    #
+    # @param (see unpack_host_key)
+    # @return (see unpack_host_key)
     def self.unpack_count(bin)
       # this may not be correct
       bin.unpack('U')[0]
     end
 
+    # Unpack string as big-endian network byte order
+    #
+    # @param (see unpack_count)
+    # @return (see unpack_count)
     def self.unpack_add_chunk_num(bin)
       bin.unpack('N')[0]
     end
 
+    # Pack a Hex String into binary
+    #
+    # @param (String) hex string to encode
+    # @return (String) encoded string
     def self.hex_to_bin(hex)
       hex.to_a.pack('H*')
     end

data/lib/google_safe_browsing/canonicalize.rb

@@ -3,11 +3,16 @@ require 'ip'
 require File.dirname(__FILE__) + '/top_level_domain.rb'
 
 module GoogleSafeBrowsing
+  # Helpers to Canonicalize urls and generate url permutations for lookups
   class Canonicalize
 
     PROTOCOL_DELIMITER = '://'
     DEFAULT_PROTOCOL = 'http'
 
+    # Base Canonicalizer method
+    #
+    # @param (String) uncanonicalized url string
+    # @return (String) canonicalized url string
     def self.url(raw_url)
       #puts raw_url
       #remove tabs, carriage returns and line feeds
@@ -37,6 +42,10 @@ module GoogleSafeBrowsing
       strict_escape(cann)
     end
 
+    # Generate the url permutations for lookup
+    #
+    # @param (String) lookup_url uncanonicalized url string
+    # @return (Array) array of cannonicalized url permutation strings
     def self.urls_for_lookup(lookup_url)
       lookup_url = url(lookup_url)
 
@@ -46,6 +55,9 @@ module GoogleSafeBrowsing
 
       host_string = strip_username_password_and_port_from_host(splits[:host])
 
+      #return empty array unless host_string has at least one period
+      return [] unless host_string.include?('.')
+
       host_strings = [host_string]
       host = TopLevelDomain.split_from_host(host_string).last(5)
       ( host.length - 1 ).times do 
@@ -61,45 +73,54 @@ module GoogleSafeBrowsing
 
     private
 
-    def self.generate_path_strings(raw_path)
-      return [ '/', '' ] if raw_path == ''
-
-      path_split = raw_path.split('?')
-      path = path_split[0]
-      params = path_split[1]
+      # Generates the path permutations from the raw path string
+      #
+      # @param (String) raw_path path split from the full url string
+      # @return (Array) array of path permutation strings
+      def self.generate_path_strings(raw_path)
+        return [ '/', '' ] if raw_path == ''
 
+        path_split = raw_path.split('?')
+        path = path_split[0]
+        params = path_split[1]
 
-      path_components = path.split('/').first(3)
-      path_strings = [ '/' ]
-      path_components.length.times do
-        path_strings << '/' + path_components.join('/')
-        path_components.pop
-      end
 
-      path_strings.map! do |p|
-        unless p.index('.')
-          p + '/'
-        else
-          p
+        path_components = path.split('/').first(3)
+        path_strings = [ '/' ]
+        path_components.length.times do
+          path_strings << '/' + path_components.join('/')
+          path_components.pop
         end
-      end
-      path_strings.map!{ |p| p.to_s.gsub!(/\/+/, '/') }
-      path_strings.compact!
-      path_strings.uniq!
 
-      if params
-        path_strings | path_strings.map do |p|
-          if p[-1..-1] == '/'
-            p
+        path_strings.map! do |p|
+          unless p.index('.')
+            p + '/'
           else
-            "#{p}?#{params}"
+            p
           end
         end
-      else
-        return path_strings
+        path_strings.map!{ |p| p.to_s.gsub!(/\/+/, '/') }
+        path_strings.compact!
+        path_strings.uniq!
+
+        if params
+          path_strings | path_strings.map do |p|
+            if p[-1..-1] == '/'
+              p
+            else
+              "#{p}?#{params}"
+            end
+          end
+        else
+          return path_strings
+        end
       end
-    end
 
+      # Returns the cartesian product of two arrays by concatination of the string representation of the elements
+      #
+      # @param (Array) a_one array of strings
+      # @param (Array) a_two array of strings
+      # @return (Array) cartesian product of arrays with elements concatinated
       def self.cart_prod(a_one, a_two)
         result = []
         a_one.each do |i|
@@ -110,6 +131,10 @@ module GoogleSafeBrowsing
         result
       end
 
+      # Takes the canonicalized url and splits the host and the path apart
+      #
+      # @param (String) cann canonicalized url string
+      # @return (Hash) !{ :host => host_part, :path => path_part }
       def self.split_host_path(cann)
         ret= { :host => cann, :path => '' }
         split_point = cann.index('/')
@@ -121,11 +146,19 @@ module GoogleSafeBrowsing
         ret
       end
 
+      # Strips the fragment portion of the url string (the last '#' and everything after)
+      #
+      # @param (String) string url
+      # @return (String) parameter with the fragment removed
       def self.remove_fragment(string)
         string = string[0..string.index('#')-1] if string.index('#')
         string
       end
 
+      # Continues to unescape the url until unescaping has no effect
+      #
+      # @param (String) url url string
+      # @return (String) fully unescaped url string
       def self.recursively_unescape(url)
         compare_url = url.clone 
         url = URI.unescape(url)
@@ -136,6 +169,10 @@ module GoogleSafeBrowsing
         url
       end
 
+      # Apply initial fixes to host string
+      #
+      # @param (String) host host string
+      # @return (String) standardized host string
       def self.fix_host(host)
         #puts "In Host: #{host}"
         # remove leading and trailing dots, multiple dots to one
@@ -149,6 +186,10 @@ module GoogleSafeBrowsing
         host
       end
 
+      # Apply initial fixes to path string
+      #
+      # @param (String) path path string
+      # @return (String) standardized path string
       def self.fix_path(path)
         #puts "In Path: #{path}"
 
@@ -179,6 +220,10 @@ module GoogleSafeBrowsing
         path
       end
 
+      # Escape the url, but do not escape certain characters; such as the carat
+      #
+      # @param (String) url url string
+      # @return (String) escaped url string
       def self.strict_escape(url)
         url = URI.escape url
 
@@ -188,6 +233,10 @@ module GoogleSafeBrowsing
         url
       end
 
+      # Strip the leading protocol from the url string
+      #
+      # @param (String) cann url string
+      # @return (String) url string without the protocol
       def self.remove_protocol(cann)
         if cann.index(PROTOCOL_DELIMITER)
           delimiting_index = cann.index(PROTOCOL_DELIMITER)
@@ -198,11 +247,19 @@ module GoogleSafeBrowsing
         cann
       end
 
+      # Strip the user name, password and port number from the url
+      #
+      # @param (String) host_string host portion of the url
+      # @return (String) host portion of the url without the username, password and port
       def self.strip_username_password_and_port_from_host(host_string)
         host_string = remove_port(host_string)
         remove_username_and_password(host_string)
       end
 
+      # Strip port number from host string
+      #
+      # @param (see strip_username_password_and_port_from_host)
+      # @return (String) host part without the port number
       def self.remove_port(host_string)
         port_sep = host_string.rindex(':')
         if port_sep
@@ -212,6 +269,10 @@ module GoogleSafeBrowsing
         end
       end
 
+      # Strip user name and password from host part of url
+      #
+      # @param (see remove_port)
+      # @return (String) host part of url without user name or password
       def self.remove_username_and_password(host_string)
         un_sep = host_string.index('@')
         if un_sep

data/lib/google_safe_browsing/top_level_domain.rb

@@ -20,6 +20,9 @@ module GoogleSafeBrowsing
       tld
     end
 
+    # return array of host components (www, example, com from www.example.com)
+    # taking into account of top level domains
+    # e.g. 'sub.domain.example.co.uk' => [ 'sub', 'domain', 'example', 'co.uk' ]
     def self.split_from_host(host)
       components = host.split('.')
 

data/lib/google_safe_browsing/version.rb

@@ -1,3 +1,3 @@
 module GoogleSafeBrowsing
-  VERSION = "0.3.0"
+  VERSION = "0.3.1"
 end

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: google_safe_browsing
 version: !ruby/object:Gem::Version 
-  hash: 19
+  hash: 17
   prerelease: 
   segments: 
   - 0
   - 3
-  - 0
-  version: 0.3.0
+  - 1
+  version: 0.3.1
 platform: ruby
 authors: 
 - Chris Marshall
@@ -15,7 +15,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2012-03-05 00:00:00 Z
+date: 2012-04-09 00:00:00 Z
 dependencies: 
 - !ruby/object:Gem::Dependency 
   prerelease: false
@@ -136,7 +136,7 @@ files:
 - MIT-LICENSE
 - Rakefile
 - README.mkd
-homepage: https://github.com/chrismar035/google_safe_browsing
+homepage: https://github.com/mobiledefense/mobiledefense_google_safe_browsing
 licenses: []
 
 post_install_message: