google_safe_browsing 0.4.2 → 0.5.0

checksums.yaml

@@ -0,0 +1,15 @@
+---
+!binary "U0hBMQ==":
+  metadata.gz: !binary |-
+    N2FiMzExYzAyYmY2NTgwZTliYjMzZmFkODJhMDkyZTg3MWU5MWUxMA==
+  data.tar.gz: !binary |-
+    YWZlYTI2Nzc5NTA0NDU0NWNjYjg2ODIyZGIxMjY5MGNjMDE1MWIxOA==
+!binary "U0hBNTEy":
+  metadata.gz: !binary |-
+    NWU5ZGNmMjI1NTBlOTZhNGNmOWZhOWZlYTg4NDQ3OTg5NmIyYzJmYjdjZTc0
+    NmEyY2RmMGM0MDg2N2FmMjJkY2QwM2U4ODM2ODRhOTgxMWYwZGJlNWQ2ODFk
+    NzI0YzVhMzlkYWNjNjNmMDg4YTcyYTkzZGI1ODc2M2IwMGQyMjI=
+  data.tar.gz: !binary |-
+    OTBmYzk0MzVhN2M2MzI4MGM0M2RhZmNkNTRiYWM3NDRlNDQ4MWE5M2E1ZTBl
+    Yzk3YzU0OWUwMzRjNWJkMDE1ZjJiMWFiNjcwYzJhMmI0NWM5NDRjOGYxMzFj
+    YWJhZDFmODgwYTI3OWI2MzhlZTk5OTAxYjZmNjQyNjYzNDc1MjU=

data/APACHE-LICENSE

@@ -0,0 +1,13 @@
+Copyright 2012 Mobile Defense
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.

data/{README.mkd → README.md}

@@ -10,6 +10,7 @@ It includes:
 * method to lookup a url
 * rake tasks to update hash database
 * Autonomous updating via Resque and Resque Scheduler (optional)
+* Message Authentication Codes (optional; on by default)
 
 ----------------------
 

data/lib/google_safe_browsing.rb

@@ -12,6 +12,8 @@ require  'google_safe_browsing/canonicalize'
 require  'google_safe_browsing/chunk_helper'
 require  'google_safe_browsing/hash_helper'
 require  'google_safe_browsing/http_helper'
+require  'google_safe_browsing/invalid_mac_validation'
+require  'google_safe_browsing/key_helper'
 require  'google_safe_browsing/response_helper'
 require  'google_safe_browsing/top_level_domain'
 
@@ -25,14 +27,21 @@ module GoogleSafeBrowsing
 
   # Handles the configuration values for the module
   class Config
-    attr_accessor :client, :app_ver, :p_ver, :host, :current_lists, :api_key
+    attr_accessor :client, :app_ver, :p_ver, :host, :current_lists, :api_key,
+      :mac_required, :client_key, :wrapped_key, :rekey_host
 
     def initialize
       @client         = 'api'
       @app_ver        = VERSION
       @p_ver          = '2.2'
       @host           = 'http://safebrowsing.clients.google.com/safebrowsing'
+      @rekey_host    = 'https://sb-ssl.google.com/safebrowsing'
       @current_lists  = [ 'googpub-phish-shavar', 'goog-malware-shavar' ]
+      @mac_required   = true
+    end
+
+    def have_keys?
+      @mac_required && @client_key.present? && @wrapped_key.present?
     end
   end
 

data/lib/google_safe_browsing/api_v2.rb

@@ -5,6 +5,8 @@ module GoogleSafeBrowsing
     #
     # @return (Integer) the number of seconds before this method should be called again
     def self.update
+      HttpHelper.get_keys unless GoogleSafeBrowsing.config.have_keys?
+
       data_response = HttpHelper.get_data
 
       to_do_array = ResponseHelper.parse_data_response(data_response.body)

data/lib/google_safe_browsing/chunk_helper.rb

@@ -1,8 +1,8 @@
 module GoogleSafeBrowsing
   class ChunkHelper
-    def self.build_chunk_list(*list)
-      lists = if list.any?
-                list.to_a
+    def self.build_chunk_list(*lists)
+      lists = if lists.any?
+                lists.to_a
               else
                 GoogleSafeBrowsing.config.current_lists
               end
@@ -21,29 +21,21 @@ module GoogleSafeBrowsing
           order(:chunk_number).uniq.collect{|c| c.chunk_number }
         action_strings << "s:#{squish_number_list(nums)}" if nums.any?
 
-        ret += "#{action_strings.join(':')}\n"
+        ret += "#{action_strings.join(':')}#{":mac" if GoogleSafeBrowsing.config.have_keys?}\n"
       end
 
-      #puts ret
       ret
     end
 
     def self.squish_number_list(chunks)
       num_strings = []
+      streak_begin = last_num = chunks.shift
 
-      streak_begin = chunks[0]
-      last_num = chunks.shift
       chunks.each do |c|
-        if c == last_num+1
-          #puts "streak continues"
-        else
-          #puts "streak has ended"
+        unless c == last_num+1
           if streak_begin != last_num
-            streak_string = "#{streak_begin}-#{last_num}"
-            #puts "there is a streak: #{streak_string}"
-            num_strings << streak_string
+            num_strings << "#{streak_begin}-#{last_num}"
           else
-            #puts "streak was one long: #{last_num}"
             num_strings << last_num
           end
           streak_begin = c

data/lib/google_safe_browsing/hash_helper.rb

@@ -21,7 +21,6 @@ module GoogleSafeBrowsing
       urls.each do |u|
         hash = ( Digest::SHA256.new << u ).to_s
         hashes << GsbHash.new(hash)
-        #puts "#{u} -- #{hash}"
       end
       hashes
     end

data/lib/google_safe_browsing/http_helper.rb

@@ -1,47 +1,126 @@
 module GoogleSafeBrowsing
   class HttpHelper
-    def self.uri_builder(action)
-      uri = URI("#{GoogleSafeBrowsing.config.host}/#{action}#{encoded_params}")
-      uri
-    end
+    def self.uri_builder(action, use_ssl=false)
+      host = GoogleSafeBrowsing.config.host
+      host = switch_to_https(host) if use_ssl
 
-    def self.encoded_params
-      "?client=#{GoogleSafeBrowsing.config.client}" +
-      "&apikey=#{GoogleSafeBrowsing.config.api_key}" +
-      "&appver=#{GoogleSafeBrowsing.config.app_ver}" +
-      "&pver=#{GoogleSafeBrowsing.config.p_ver}"
+      uri = URI("#{host}/#{action}#{encoded_params}")
+      uri
     end
 
     def self.request_full_hashes(hash_array)
+      get_keys unless GoogleSafeBrowsing.config.have_keys?
+
       uri = uri_builder('gethash')
-      request = Net::HTTP::Post.new(uri.request_uri)
-      request.body = "4:#{hash_array.length * 4}\n"
-      hash_array.each do |h|
-        request.body << BinaryHelper.hex_to_bin(h[0..7])
-      end
 
-      response = Net::HTTP.start(uri.host) { |http| http.request request }
+      response = post_data(uri) do
+        body = "4:#{hash_array.length * 4}\n"
+        hash_array.each do |h|
+          body << BinaryHelper.hex_to_bin(h[0..7])
+        end
+
+        body
+      end
 
-      if response.is_a?(Net::HTTPSuccess) && !response.body.blank? # we are seeing blank responses from Google
+      if response.is_a?(Net::HTTPSuccess) && !response.body.blank?
         ResponseHelper.parse_full_hash_response(response.body)
       else
-        # if response not good, return empty array to represent no full hashes
         []
       end
     end
 
     def self.get_data(list=nil)
-      # Get (via Post) List Data
       uri = uri_builder('downloads')
-      request = Net::HTTP::Post.new(uri.request_uri)
-      request.body = ChunkHelper.build_chunk_list(list)
 
-      Net::HTTP.start(uri.host) { |http| http.request request }
+      post_data(uri) do
+        ChunkHelper.build_chunk_list(list)
+      end
     end
 
-    def get_lists
-      uri = uri_builder('list')
-      Net::HTTP.get(uri).split("\n")
+    def self.get_keys
+      uri = URI("#{GoogleSafeBrowsing.config.rekey_host}/newkey#{encoded_params}")
+
+      http = Net::HTTP.new(uri.host, uri.port)
+      http.use_ssl = true
+      http.verify_mode = OpenSSL::SSL::VERIFY_NONE
+
+      request = Net::HTTP::Get.new(uri.request_uri)
+      response = http.request(request)
+      response.body.split("\n").each do |key_line|
+        key_name, _, key_value = key_line.split(':')
+        key_value.gsub!('=', '')
+
+        case key_name
+        when 'clientkey'
+          key_value = KeyHelper::web_safe_base64_decode(key_value)
+          GoogleSafeBrowsing.config.client_key = key_value
+        when 'wrappedkey'
+          GoogleSafeBrowsing.config.wrapped_key = key_value
+        end
+      end
     end
+
+    private
+      REKEY_PREFIX = 'e:pleaserekey'
+
+      def self.encoded_params
+        params = "?client=#{GoogleSafeBrowsing.config.client}" <<
+        "&apikey=#{GoogleSafeBrowsing.config.api_key}" <<
+        "&appver=#{GoogleSafeBrowsing.config.app_ver}" <<
+        "&pver=#{GoogleSafeBrowsing.config.p_ver}"
+
+        params << "&wrkey=#{GoogleSafeBrowsing.config.wrapped_key}" if GoogleSafeBrowsing.config.have_keys?
+
+        params
+      end
+
+      def self.with_keys(uri)
+        begin
+          get_keys unless GoogleSafeBrowsing.config.have_keys?
+          response = yield uri
+        end while self.please_rekey?(response.body)
+
+        lines = response.body.split("\n")
+        mac = lines.shift
+        if mac[0..1] == 'm:'
+          mac = mac[2..-1].chomp
+          data = lines.join("\n") << "\n"
+        else
+          data = lines.join("\n")
+        end
+
+        if self.valid_mac?(data, mac)
+          response
+        else
+          raise InvalidMACValidation, "The MAC returned from '#{uri}' is not valid."
+        end
+      end
+
+      def self.valid_mac?(data, mac)
+        KeyHelper.compute_mac_code(data) == mac
+      end
+
+      def self.post_data(uri)
+        with_keys uri do
+          request = Net::HTTP::Post.new(uri.request_uri)
+          request.body = yield uri
+
+          Net::HTTP.start(uri.host) { |http| http.request request }
+        end
+      end
+
+      def self.please_rekey?(body)
+        if body.split("\n").include? REKEY_PREFIX
+          GoogleSafeBrowsing.config.client_key = nil
+          GoogleSafeBrowsing.config.wrapped_key = nil
+          true
+        else
+          false
+        end
+      end
+
+      def self.switch_to_https(url)
+        "https#{url[4..-1]}"
+      end
   end
 end

data/lib/google_safe_browsing/invalid_mac_validation.rb

@@ -0,0 +1,2 @@
+class InvalidMACValidation < StandardError
+end

data/lib/google_safe_browsing/key_helper.rb

@@ -0,0 +1,23 @@
+module GoogleSafeBrowsing
+  class KeyHelper
+
+    def self.web_safe_base64_decode(str)
+      str.tr!('-_', '+/')
+      str << '=' while str.length % 4 != 0
+      Base64.decode64(str)
+    end
+
+    def self.web_safe_base64_encode(str)
+      str = Base64.encode64(str).chomp
+      str.tr('+/', '-_')
+    end
+
+    def self.compute_mac_code(data)
+      sha1 = OpenSSL::HMAC.digest('sha1',
+                                  GoogleSafeBrowsing.config.client_key,
+                                  data)
+      web_safe_base64_encode sha1
+    end
+  end
+end
+

data/lib/google_safe_browsing/response_helper.rb

@@ -18,8 +18,6 @@ module GoogleSafeBrowsing
     end
 
     def self.parse_data_response(response)
-      #print "\n\n#{response}\n\n"
-      data_urls = []
       ret = {}
 
       ret[:lists] = []
@@ -43,25 +41,34 @@ module GoogleSafeBrowsing
         when 'ad'
           # vals[1] is a CHUNKLIST number or range representing add chunks to delete
           # we can also delete the associated Shavar Hashes
-          # we no longer have to report hat we received these chunks
+          # we no longer have to report that we received these chunks
           chunk_number_clause = ChunkHelper.chunklist_to_sql(vals[1])
-          AddShavar.delete_all([ "list = ? and (#{chunk_number_clause})", current_list ])
+          AddShavar.delete_all([ "list = ? and (?)", current_list, chunk_number_clause ])
         when 'sd'
           # vals[1] is a CHUNKLIST number or range representing sub chunks to delete
-          # we no longer have to report hat we received these chunks
+          # we no longer have to report that we received these chunks
           chunk_number_clause = ChunkHelper.chunklist_to_sql(vals[1])
-          SubShavar.delete_all([ "list = ? and (#{chunk_number_clause})", current_list ])
+          SubShavar.delete_all([ "list = ? and (?)", current_list, chunk_number_clause ])
         end
       end
 
-      #ret[:data_urls] = data_urls
-
       ret
     end
 
     def self.receive_data(url, list)
+      urls = url.split(',')
+      url = urls[0]
+      mac = urls[1]
 
       open(url) do |f|
+        body = f.read
+
+        unless mac.blank? || HttpHelper.valid_mac?(body, mac)
+          raise InvalidMACVerification
+        end
+
+        f.rewind
+
         while(line = f.gets)
           line_actions = parse_data_line(line)
 
@@ -133,31 +140,48 @@ module GoogleSafeBrowsing
         end
       end
 
-      # actually perform inserts
       while @add_shavar_values && @add_shavar_values.any?
-        AddShavar.connection.execute( "insert into gsb_add_shavars (prefix, host_key, chunk_number, list) " +
-                                      " values #{@add_shavar_values.pop(10000).join(', ')}") 
+        AddShavar.connection.execute <<-SQL
+          INSERT INTO gsb_add_shavars (prefix, host_key, chunk_number, list)
+          VALUES #{pop_and_join @add_shavar_values}
+          SQL
       end
       while @sub_shavar_values && @sub_shavar_values.any?
-      SubShavar.connection.execute( "insert into gsb_sub_shavars (prefix, host_key, add_chunk_number, chunk_number, list) " +
-                                    " values #{@sub_shavar_values.pop(10000).join(', ')}") 
+      SubShavar.connection.execute <<-SQL
+        INSERT INTO gsb_sub_shavars (prefix,
+                                     host_key,
+                                     add_chunk_number,
+                                     chunk_number,
+                                     list)
+        VALUES #{pop_and_join @sub_shavar_values}
+        SQL
       end
-      # remove invalid full_hases
-      FullHash.connection.execute("delete from gsb_full_hashes using gsb_full_hashes " + 
-                                  "inner join  gsb_sub_shavars on " +
-                                  "gsb_sub_shavars.add_chunk_number = gsb_full_hashes.add_chunk_number " +
-                                  "and gsb_sub_shavars.list = gsb_full_hashes.list;")
+
+      FullHash.connection.execute <<-SQL
+        DELETE FROM gsb_full_hashes
+        USING gsb_full_hashes
+        INNER JOIN  gsb_sub_shavars ON
+        gsb_sub_shavars.add_chunk_number = gsb_full_hashes.add_chunk_number
+        AND gsb_sub_shavars.list = gsb_full_hashes.list;
+        SQL
+
       @add_shavar_values = []
       @sub_shavar_values = []
     end
 
     def self.record_add_shavar_to_insert(h)
       @add_shavar_values ||= []
-      @add_shavar_values << "('#{h[:prefix]}', '#{h[:host_key]}', '#{h[:chunk_number]}', '#{h[:list]}')"
+      values = [ h[:prefix], h[:host_key], h[:chunk_number], h[:list] ]
+      @add_shavar_values << "(#{escape_and_join values})"
     end
     def self.record_sub_shavar_to_insert(h)
       @sub_shavar_values ||= []
-      @sub_shavar_values << "('#{h[:prefix]}', '#{h[:host_key]}', '#{h[:add_chunk_number]}', '#{h[:chunk_number]}', '#{h[:list]}')"
+      values = [ h[:prefix],
+                 h[:host_key],
+                 h[:add_chunk_number],
+                 h[:chunk_number],
+                 h[:list] ]
+      @sub_shavar_values << "(#{escape_and_join values})"
     end
 
     def self.parse_data_line(line)
@@ -168,14 +192,17 @@ module GoogleSafeBrowsing
       ret[ :chunk_number ]  = split_line[1].to_i
       ret[ :hash_length ]   = split_line[2].to_i
       ret[ :chunk_length ]  = split_line[3].to_i
-
-      #puts "Chunk ##{s_chunk_count + a_chunk_count}"
-      #puts "Action: #{action}"
-      #puts "Chunk Number: #{split_line[1]}"
-      #puts "Hash Length: #{hash_length}"
-      #puts "Chunk Length: #{chunk_length}"
-      ##puts "Chuch Data:\n#{chunk}\nend"
       ret
     end
+
+    def self.escape_and_join(values)
+      values.map do |v|
+        ActiveRecord::Base::sanitize(v)
+      end.join(', ')
+    end
+
+    def self.pop_and_join(records)
+      records.pop(10000).join(', ')
+    end
   end
 end

data/lib/google_safe_browsing/version.rb

@@ -1,3 +1,3 @@
 module GoogleSafeBrowsing
-  VERSION = "0.4.2"
+  VERSION = "0.5.0"
 end

metadata

@@ -1,20 +1,18 @@
 --- !ruby/object:Gem::Specification
 name: google_safe_browsing
 version: !ruby/object:Gem::Version
-  version: 0.4.2
-  prerelease: 
+  version: 0.5.0
 platform: ruby
 authors:
 - Chris Marshall
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-08-27 00:00:00.000000000 Z
+date: 2013-05-01 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rails
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
@@ -22,7 +20,6 @@ dependencies:
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
@@ -30,7 +27,6 @@ dependencies:
 - !ruby/object:Gem::Dependency
   name: ruby-ip
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
@@ -38,7 +34,6 @@ dependencies:
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
@@ -46,7 +41,6 @@ dependencies:
 - !ruby/object:Gem::Dependency
   name: activerecord
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
@@ -54,7 +48,6 @@ dependencies:
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
@@ -62,7 +55,6 @@ dependencies:
 - !ruby/object:Gem::Dependency
   name: sqlite3
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
@@ -70,7 +62,6 @@ dependencies:
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
@@ -78,7 +69,6 @@ dependencies:
 - !ruby/object:Gem::Dependency
   name: rspec-rails
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
@@ -86,7 +76,6 @@ dependencies:
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
@@ -94,7 +83,6 @@ dependencies:
 - !ruby/object:Gem::Dependency
   name: generator_spec
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
@@ -102,7 +90,20 @@ dependencies:
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: webmock
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
@@ -127,6 +128,8 @@ files:
 - lib/google_safe_browsing/google_safe_browsing_railtie.rb
 - lib/google_safe_browsing/hash_helper.rb
 - lib/google_safe_browsing/http_helper.rb
+- lib/google_safe_browsing/invalid_mac_validation.rb
+- lib/google_safe_browsing/key_helper.rb
 - lib/google_safe_browsing/rescheduler.rb
 - lib/google_safe_browsing/response_helper.rb
 - lib/google_safe_browsing/sub_shavar.rb
@@ -134,37 +137,30 @@ files:
 - lib/google_safe_browsing/version.rb
 - lib/google_safe_browsing.rb
 - lib/tasks/google_safe_browsing_tasks.rake
-- MIT-LICENSE
+- APACHE-LICENSE
 - Rakefile
-- README.mkd
+- README.md
 homepage: https://github.com/mobiledefense/mobiledefense_google_safe_browsing
 licenses: []
+metadata: {}
 post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
-  none: false
   requirements:
   - - ! '>='
     - !ruby/object:Gem::Version
       version: '0'
-      segments:
-      - 0
-      hash: 809089230686412521
 required_rubygems_version: !ruby/object:Gem::Requirement
-  none: false
   requirements:
   - - ! '>='
     - !ruby/object:Gem::Version
       version: '0'
-      segments:
-      - 0
-      hash: 809089230686412521
 requirements: []
 rubyforge_project: 
-rubygems_version: 1.8.24
+rubygems_version: 2.0.0
 signing_key: 
-specification_version: 3
+specification_version: 4
 summary: Rails 3 plugin for Google's Safe Browsing API v2
 test_files: []

data/MIT-LICENSE

@@ -1,20 +0,0 @@
-Copyright 2012 Chris Marshall
-
-Permission is hereby granted, free of charge, to any person obtaining
-a copy of this software and associated documentation files (the
-"Software"), to deal in the Software without restriction, including
-without limitation the rights to use, copy, modify, merge, publish,
-distribute, sublicense, and/or sell copies of the Software, and to
-permit persons to whom the Software is furnished to do so, subject to
-the following conditions:
-
-The above copyright notice and this permission notice shall be
-included in all copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
-EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
-MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
-NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
-LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
-OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
-WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.