google_safe_browsing 0.1.0

data/lib/google_safe_browsing/full_hash.rb

@@ -0,0 +1,5 @@
+module GoogleSafeBrowsing
+  class FullHash < ActiveRecord::Base
+    set_table_name 'gsb_full_hashes'
+  end
+end

data/lib/google_safe_browsing/google_safe_browsing_railtie.rb

@@ -0,0 +1,19 @@
+module GoogleSafeBrowsing
+  class GoogleSafeBrowsingRailtie < Rails::Railtie
+    config.google_safe_browsing = ActiveSupport::OrderedOptions.new
+
+    rake_tasks do
+      load File.expand_path('../../tasks/google_safe_browsing_tasks.rake', __FILE__)
+    end
+
+    generators do
+      require File.expand_path('../../generators/install_generator', __FILE__)
+    end
+
+    initializer 'google_safe_browsing.set_api_key' do |app|
+      GoogleSafeBrowsing.configure do |config|
+        config.api_key = app.config.google_safe_browsing[:api_key]
+      end
+    end
+  end
+end

data/lib/google_safe_browsing/hash_helper.rb

@@ -0,0 +1,29 @@
+module GoogleSafeBrowsing
+  class HashHelper
+
+    class GsbHash
+      def initialize(hash)
+        @hash = hash
+      end
+
+      def prefix
+        @hash[0..7]
+      end
+
+      def to_s
+        @hash
+      end
+
+    end
+
+    def self.urls_to_hashes(urls)
+      hashes = []
+      urls.each do |u|
+        hash = ( Digest::SHA256.new << u ).to_s
+        hashes << GsbHash.new(hash)
+        #puts "#{u} -- #{hash}"
+      end
+      hashes
+    end
+  end
+end

data/lib/google_safe_browsing/http_helper.rb

@@ -0,0 +1,42 @@
+module GoogleSafeBrowsing
+  class HttpHelper
+    def self.uri_builder(action)
+      uri = URI("#{GoogleSafeBrowsing.config.host}/#{action}#{encoded_params}")
+      uri
+    end
+
+    def self.encoded_params
+      "?client=#{GoogleSafeBrowsing.config.client}" +
+      "&apikey=#{GoogleSafeBrowsing.config.api_key}" +
+      "&appver=#{GoogleSafeBrowsing.config.app_ver}" +
+      "&pver=#{GoogleSafeBrowsing.config.p_ver}"
+    end
+
+    def self.request_full_hashes(hash_array)
+      uri = uri_builder('gethash')
+      request = Net::HTTP::Post.new(uri.request_uri)
+      request.body = "4:#{hash_array.length * 4}\n"
+      hash_array.each do |h|
+        request.body << BinaryHelper.hex_to_bin(h[0..7])
+      end
+
+      response = Net::HTTP.start(uri.host) { |http| http.request request }
+
+      ResponseHelper.parse_full_hash_response(response.body)
+    end
+
+    def self.get_data(list=nil)
+      # Get (via Post) List Data
+      uri = uri_builder('downloads')
+      request = Net::HTTP::Post.new(uri.request_uri)
+      request.body = ChunkHelper.build_chunk_list(list)
+
+      Net::HTTP.start(uri.host) { |http| http.request request }
+    end
+
+    def get_lists
+      uri = uri_builder('list')
+      lists = Net::HTTP.get(uri).split("\n")
+    end
+  end
+end

data/lib/google_safe_browsing/rescheduler.rb

@@ -0,0 +1,14 @@
+module GoogleSafeBrowsing
+  class Rescheduler
+    @queue = :google_safe_browsing
+
+    def self.perform
+      puts "Running Update"
+      delay = APIv2.update
+      puts "Scheduling new update in #{delay} seconds"
+      Resque.enqueue_in(delay.seconds, Rescheduler)
+      puts "Update scheduled"
+    end
+
+  end
+end

data/lib/google_safe_browsing/response_helper.rb

@@ -0,0 +1,181 @@
+module GoogleSafeBrowsing
+  class ResponseHelper
+    def self.parse_full_hash_response(response)
+      f = StringIO.new(response)
+
+      full_hashes = []
+      while(! f.eof? )
+        hash = {}
+
+        meta = f.gets.chomp.split(':')
+        hash[:list] = meta[0]
+        hash[:add_chunk_num] = meta[1]
+
+        hash[:full_hash] = f.read(meta[2].to_i).unpack('H*')[0]
+        full_hashes << hash
+      end
+      full_hashes
+    end
+
+    def self.parse_data_response(response)
+      #print "\n\n#{response}\n\n"
+      data_urls = []
+      ret = {}
+
+      ret[:lists] = []
+      ret[:data_urls] = {}
+
+      current_list = ''
+      response.split("\n").each do |line|
+        vals = line.split(':')
+        case vals[0]
+        when 'n'
+          ret[:delay_seconds] = vals[1].to_i
+          @delay_seconds = ret[:delay_seconds].to_i
+        when 'i'
+          ret[:lists] << vals[1]
+          current_list = vals[1]
+          ret[:data_urls][current_list] = []
+        when 'u'
+          ret[:data_urls][current_list] << vals[1]
+        when 'r'
+          # reset (delete all data and try again)
+        when 'ad'
+          # vals[1] is a CHUNKLIST number or range representing add chunks to delete
+          # we can also delete the associated Shavar Hashes
+          # we no longer have to report hat we received these chunks
+          chunk_number_clause = ChunkHelper.chunklist_to_sql(vals[1])
+          AddShavar.delete_all([ "list = ? and (#{chunk_number_clause})", current_list ])
+        when 'sd'
+          # vals[1] is a CHUNKLIST number or range representing sub chunks to delete
+          # we no longer have to report hat we received these chunks
+          chunk_number_clause = ChunkHelper.chunklist_to_sql(vals[1])
+          SubShavar.delete_all([ "list = ? and (#{chunk_number_clause})", current_list ])
+        end
+      end
+
+      #ret[:data_urls] = data_urls
+
+      ret
+    end
+
+    def self.receive_data(url, list)
+
+      open(url) do |f|
+        while(line = f.gets)
+          line_actions = parse_data_line(line)
+
+          chunk  = f.read(line_actions[:chunk_length])
+          # f iterator is now set for next chunk
+
+          add_attrs = { :chunk_number => line_actions[:chunk_number],
+                    :list => list, :prefix => nil, :host_key => nil
+                  }
+
+          case line_actions[:action]
+          when 'a'
+            if line_actions[:chunk_length] == 0
+              record_add_shavar_to_insert(add_attrs)
+            else
+              chunk_iterator = chunk.bytes
+              counter = 0
+              begin
+                while true
+                  add_attrs[:host_key] = BinaryHelper.read_bytes_as_hex(chunk_iterator, 4)
+                  count = chunk_iterator.next
+                  if count > 0
+                    count.times do |i|
+                      add_attrs[:prefix] = BinaryHelper.read_bytes_as_hex(chunk_iterator, line_actions[:hash_length])
+                      record_add_shavar_to_insert(add_attrs)
+                    end
+                  else
+                    add_attrs[:prefix] = add_attrs[:host_key]
+                    record_add_shavar_to_insert(add_attrs)
+                  end
+                  counter += 1
+                end
+              rescue StopIteration
+                puts "Added #{counter} host_keys for add chunk number #{line_actions[:chunk_number]}"
+              end
+            end
+          when 's'
+            sub_attrs = add_attrs.merge({ :add_chunk_number => nil })
+            if line_actions[:chunk_length] == 0
+              record_sub_shavar_to_insert(sub_attrs)
+            else
+              chunk_iterator = chunk.bytes
+              counter = 0
+              begin
+                while true
+                  sub_attrs[:host_key] = BinaryHelper.read_bytes_as_hex(chunk_iterator, 4)
+                  count = chunk_iterator.next
+                  if count > 0
+                    count.times do |i|
+                      sub_attrs[:add_chunk_number] = BinaryHelper.unpack_add_chunk_num(BinaryHelper.read_bytes_from(chunk_iterator, 4))
+                      sub_attrs[:prefix] = BinaryHelper.read_bytes_as_hex(chunk_iterator, line_actions[:hash_length])
+                      record_sub_shavar_to_insert(sub_attrs)
+                    end
+                  else
+                    sub_attrs[:add_chunk_number] = BinaryHelper.unpack_add_chunk_num(BinaryHelper.read_bytes_from(chunk_iterator, 4))
+                    sub_attrs[:prefix] = sub_attrs[:host_key]
+                    record_sub_shavar_to_insert(sub_attrs)
+                  end
+                  counter += 1
+                end
+              rescue StopIteration
+                puts "Added #{counter} host_keys for sub chunk number #{line_actions[:chunk_number]}"
+              end
+            end
+          else
+            puts "neither a nor s ======================================================="
+          end
+
+        end
+      end
+
+      # actually perform inserts
+      while @add_shavar_values && @add_shavar_values.any?
+        AddShavar.connection.execute( "insert into gsb_add_shavars (prefix, host_key, chunk_number, list) " +
+                                      " values #{@add_shavar_values.pop(10000).join(', ')}") 
+      end
+      while @sub_shavar_values && @sub_shavar_values.any?
+      SubShavar.connection.execute( "insert into gsb_sub_shavars (prefix, host_key, add_chunk_number, chunk_number, list) " +
+                                    " values #{@sub_shavar_values.pop(10000).join(', ')}") 
+      end
+      # remove invalid full_hases
+      FullHash.connection.execute("delete from gsb_full_hashes using gsb_full_hashes " + 
+                                  "inner join  gsb_sub_shavars on " +
+                                  "gsb_sub_shavars.add_chunk_number = gsb_full_hashes.add_chunk_number " +
+                                  "and gsb_sub_shavars.list = gsb_full_hashes.list;")
+      @add_shavar_values = []
+      @sub_shavar_values = []
+    end
+
+    def self.record_add_shavar_to_insert(h)
+      @add_shavar_values ||= []
+      @add_shavar_values << "('#{h[:prefix]}', '#{h[:host_key]}', '#{h[:chunk_number]}', '#{h[:list]}')"
+    end
+    def self.record_sub_shavar_to_insert(h)
+      @sub_shavar_values ||= []
+      @sub_shavar_values << "('#{h[:prefix]}', '#{h[:host_key]}', '#{h[:add_chunk_number]}', '#{h[:chunk_number]}', '#{h[:list]}')"
+    end
+
+    def self.parse_data_line(line)
+      split_line = line.split(':')
+
+      ret = {}
+      ret[ :action ]        = split_line[0]
+      ret[ :chunk_number ]  = split_line[1].to_i
+      ret[ :hash_length ]   = split_line[2].to_i
+      ret[ :chunk_length ]  = split_line[3].to_i
+
+      #puts "Chunk ##{s_chunk_count + a_chunk_count}"
+      #puts "Action: #{action}"
+      #puts "Chunk Number: #{split_line[1]}"
+      #puts "Hash Length: #{hash_length}"
+      #puts "Chunk Length: #{chunk_length}"
+      ##puts "Chuch Data:\n#{chunk}\nend"
+      ret
+    end
+  end
+end

data/lib/google_safe_browsing/sub_shavar.rb

@@ -0,0 +1,6 @@
+module GoogleSafeBrowsing
+  class SubShavar < ActiveRecord::Base
+    set_table_name 'gsb_sub_shavars'
+  end
+end
+

data/lib/google_safe_browsing/top_level_domain.rb

@@ -0,0 +1,50 @@
+module GoogleSafeBrowsing
+  class TopLevelDomain
+
+    def self.from_host(host)
+      components = host.split('.')
+
+      tlds = parse_tld_to_hash
+
+      tld = components.pop
+      components.reverse.each do |comp|
+        next_tld = "#{comp}.#{tld}"
+
+        if tlds[next_tld]
+          tld = next_tld
+        else
+          break
+        end
+      end
+
+      tld
+    end
+
+    def self.split_from_host(host)
+      components = host.split('.')
+
+      tlds = parse_tld_to_hash
+
+      next_tld = components[-2..-1].join('.')
+      while tlds[next_tld]
+        tmp = components.pop
+        components[-1] = components.last + '.' + tmp
+        next_tld = components[-2..-1].join('.')
+      end
+
+      components
+    end
+
+
+    private 
+
+    def self.parse_tld_to_hash
+      hash = Hash.new(nil)
+      f = File.open(File.dirname(__FILE__) + '/effective_tld_names.dat.txt', 'r')
+      while(line = f.gets)
+        hash[line.chomp] = true unless line[0..1] == '//'
+      end
+      hash
+    end
+  end
+end

data/lib/google_safe_browsing/version.rb

@@ -0,0 +1,3 @@
+module GoogleSafeBrowsing
+  VERSION = "0.1.0"
+end

data/lib/tasks/google_safe_browsing_tasks.rake

@@ -0,0 +1,11 @@
+namespace :google_safe_browsing do
+  desc "Performs an Update"
+  task :update => :environment do
+    GoogleSafeBrowsing::APIv2.update
+  end
+
+  desc "Enqueues an Update via Rescheduler, which will reschedule another Update the appropriate number of seconds in the future"
+  task :update_and_reschedule => :environment do
+    GoogleSafeBrowsing.kick_off
+  end
+end

metadata

@@ -0,0 +1,173 @@
+--- !ruby/object:Gem::Specification 
+name: google_safe_browsing
+version: !ruby/object:Gem::Version 
+  hash: 27
+  prerelease: 
+  segments: 
+  - 0
+  - 1
+  - 0
+  version: 0.1.0
+platform: ruby
+authors: 
+- Chris Marshall
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2012-02-28 00:00:00 Z
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  prerelease: false
+  version_requirements: &id001 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ~>
+      - !ruby/object:Gem::Version 
+        hash: 3
+        segments: 
+        - 3
+        - 1
+        - 0
+        version: 3.1.0
+  requirement: *id001
+  name: rails
+  type: :runtime
+- !ruby/object:Gem::Dependency 
+  prerelease: false
+  version_requirements: &id002 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ~>
+      - !ruby/object:Gem::Version 
+        hash: 57
+        segments: 
+        - 0
+        - 9
+        - 1
+        version: 0.9.1
+  requirement: *id002
+  name: ruby-ip
+  type: :runtime
+- !ruby/object:Gem::Dependency 
+  prerelease: false
+  version_requirements: &id003 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 3
+        segments: 
+        - 0
+        version: "0"
+  requirement: *id003
+  name: activerecord
+  type: :runtime
+- !ruby/object:Gem::Dependency 
+  prerelease: false
+  version_requirements: &id004 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 3
+        segments: 
+        - 0
+        version: "0"
+  requirement: *id004
+  name: sqlite3
+  type: :development
+- !ruby/object:Gem::Dependency 
+  prerelease: false
+  version_requirements: &id005 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 3
+        segments: 
+        - 0
+        version: "0"
+  requirement: *id005
+  name: rspec-rails
+  type: :development
+- !ruby/object:Gem::Dependency 
+  prerelease: false
+  version_requirements: &id006 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 3
+        segments: 
+        - 0
+        version: "0"
+  requirement: *id006
+  name: generator_spec
+  type: :development
+description: Rails 3 plugin using Google's Safe Browsing API for url lookup against Malware and Phishing blacklists. Implementation includes storing and updating locally stored shavar lists and url lookup methods.
+email: chris@chrismar035.com
+executables: []
+
+extensions: []
+
+extra_rdoc_files: []
+
+files: 
+- lib/generators/install_generator.rb
+- lib/generators/templates/create_google_safe_browsing_tables.rb
+- lib/google_safe_browsing/add_shavar.rb
+- lib/google_safe_browsing/api_v2.rb
+- lib/google_safe_browsing/binary_helper.rb
+- lib/google_safe_browsing/canonicalize.rb
+- lib/google_safe_browsing/chunk_helper.rb
+- lib/google_safe_browsing/effective_tld_names.dat.txt
+- lib/google_safe_browsing/full_hash.rb
+- lib/google_safe_browsing/google_safe_browsing_railtie.rb
+- lib/google_safe_browsing/hash_helper.rb
+- lib/google_safe_browsing/http_helper.rb
+- lib/google_safe_browsing/rescheduler.rb
+- lib/google_safe_browsing/response_helper.rb
+- lib/google_safe_browsing/sub_shavar.rb
+- lib/google_safe_browsing/top_level_domain.rb
+- lib/google_safe_browsing/version.rb
+- lib/google_safe_browsing.rb
+- lib/tasks/google_safe_browsing_tasks.rake
+- MIT-LICENSE
+- Rakefile
+- README.mkd
+homepage: https://github.com/chrismar035/google_safe_browsing
+licenses: []
+
+post_install_message: 
+rdoc_options: []
+
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 1.8.16
+signing_key: 
+specification_version: 3
+summary: Rails 3 plugin for Google's Safe Browsing API v2
+test_files: []
+