google-safe-browsing-plugin 0.1.1

data/lib/rails/generators/google/rspec/rspec_generator.rb

@@ -0,0 +1,28 @@
+# encoding: utf-8
+
+module Google
+  module Generators
+    class RspecGenerator < Rails::Generators::Base
+      desc 'Creates a Google Safe Browsing plugin Rspec test files under rspect/google'
+
+      source_root File.expand_path('../templates', __FILE__)
+
+      def copy_spec_files
+        %w(
+          bin_sample_1.data
+          bin_sample_2.data
+          full_hash_response_0.data
+          full_hash_response_1.data
+          full_hash_response_2.data
+          full_hash_response_3.data
+          full_hash_parse_spec.rb
+          shavar_encode_data_parse_spec.rb
+          shavar_list_info_parse_spec.rb
+        ).each do |f|
+          copy_file "#{f}", "spec/google/#{f}"
+        end
+      end
+
+    end
+  end
+end

data/lib/rails/generators/google/rspec/templates/full_hash_parse_spec.rb

@@ -0,0 +1,58 @@
+require 'spec_helper'
+
+describe "Parse full length hash response from GoogleSafeBrowsing" do
+  
+  include Google::SafeBrowsingParser
+
+  def read_full_length_hash_file file
+    resp = File.read(file, :encoding => 'ASCII-8BIT')
+  end
+
+  def display_list full_list
+    full_list.keys.each do |list|
+      puts "List #{list}"
+      puts "    Full length hash:"
+      full_list[list].keys.each do |add_chunk_num|
+        puts "       Add chunk num: #{add_chunk_num}"
+        full_list[list][add_chunk_num].each do |full_hash|
+          puts "             full hash: #{full_hash}"
+        end
+      end
+      puts "========="
+    end
+  end
+  
+  it "should capture single full length hash" do
+    resp = read_full_length_hash_file File.expand_path('../full_hash_response_0.data', __FILE__)
+    full_list = parse_full_hash_entries resp
+    full_list[:'goog-malware-shavar'][113869].should eq(['34f0021bea5c18b405cf9a279f91555100efc4edfb8909d074eab584fd702826'])
+  end
+
+  it "should capture 2 full length hashes" do
+    resp = read_full_length_hash_file File.expand_path('../full_hash_response_1.data', __FILE__)
+    full_list = parse_full_hash_entries resp
+    full_list[:'goog-malware-shavar'][113869].should eq(
+       ['746e32e9833b573db34c8044f6020b9379632558c4267d811cd8db70016300b7', 
+        'ad1701117dd7dcd36d8add796f0ba16c3e1fb22a0154883c9eac14f53e5a063d'])
+
+    # display_list full_list
+  end
+
+  it "should capture full length hash for both malware and phishing" do
+    resp = read_full_length_hash_file File.expand_path('../full_hash_response_2.data', __FILE__)
+    full_list = parse_full_hash_entries resp
+    full_list[:'goog-malware-shavar'][113423].should eq(['b3e357a66e2130ecb4afcb0fcefd32ff5e536c9fd3219681c045bcd48b89be3e'])
+    full_list[:'googpub-phish-shavar'][238266].should eq(['5b3583c05bdaeb39718b1d865f012663fe22f9e8de01ef0ede8d4653d87e766a'])
+    
+  end
+
+  it "should capture full length hash for more complex match" do
+    resp = read_full_length_hash_file File.expand_path('../full_hash_response_3.data', __FILE__)
+    full_list = parse_full_hash_entries resp
+    full_list[:'goog-malware-shavar'][113423].should eq(['b3e357a66e2130ecb4afcb0fcefd32ff5e536c9fd3219681c045bcd48b89be3e'])
+    full_list[:'googpub-phish-shavar'][238266].should eq([
+      '5b3583c05bdaeb39718b1d865f012663fe22f9e8de01ef0ede8d4653d87e766a',
+      '7e930bdaee322d25e4f422cc6dc688f50b1920127d8a604efd5e5ffb18d78284'])
+  end
+
+end

data/lib/rails/generators/google/rspec/templates/full_hash_response_2.data

@@ -0,0 +1,3 @@
+goog-malware-shavar:113423:32
+��W�n!0촯���2�^Sl��!���E�ԋ��>googpub-phish-shavar:238266:32
+[5��[��9q��_
&c�"���
�ލFS�~vj

data/lib/rails/generators/google/rspec/templates/full_hash_response_3.data

@@ -0,0 +1,3 @@
+goog-malware-shavar:113423:32
+��W�n!0촯���2�^Sl��!���E�ԋ��>googpub-phish-shavar:238266:64
+[5��[��9q��_
&c�"���
�ލFS�~vj~���2-%��"�mƈ� }�`N�^_�ׂ�

data/lib/rails/generators/google/rspec/templates/shavar_encode_data_parse_spec.rb

@@ -0,0 +1,56 @@
+require 'spec_helper'
+
+describe "Parse the binary data parsing" do
+
+  include Google::SafeBrowsingParser
+
+  def read_shavar_download_file file
+    resp = File.read(file, :encoding => 'ASCII-8BIT')
+  end
+  
+  it "should read add chunk binary data" do
+    file = File.expand_path('../bin_sample_1.data', __FILE__)
+  
+    resp = read_shavar_download_file file
+    adds, subs = parse_shavar_list(resp)
+    
+    add = adds.first
+    add[:chunk_num].should eq(113601)
+    add[:hash_len].should eq(4)
+    add[:chunk_len].should eq(6246)
+    data = add[:chunk_data]
+    data['3c8fbb74'].should eq(["87c99352", "3a71cbc0", "421e54f4"])
+    data['1487ee7d'].should eq([])
+
+    add = adds.last
+    add[:chunk_num].should eq(113656)
+    add[:hash_len].should eq(4)
+    add[:chunk_len].should eq(850)
+    data = add[:chunk_data]
+    data['c21ddace'].should eq([])
+    data['b3b36f29'].should eq(['5d472258'])
+
+  end
+
+  it "should read add chunk binary data" do
+    file = File.expand_path('../bin_sample_2.data', __FILE__)
+  
+    resp = read_shavar_download_file file
+    adds, subs = parse_shavar_list(resp)
+    
+    sub = subs.first
+    sub[:chunk_num].should eq(112801)
+    sub[:hash_len].should eq(4)
+    sub[:chunk_len].should eq(157)
+    data = sub[:chunk_data]
+    data['f48787fc'][108559].should eq(["37f0e2d6"])
+
+    sub = subs.last
+    sub[:chunk_num].should eq(112960)
+    sub[:hash_len].should eq(4)
+    sub[:chunk_len].should eq(210)
+    data = sub[:chunk_data]
+    data['812324f0'][111330].should eq(["981f1261"])
+  end
+
+end  

data/lib/rails/generators/google/rspec/templates/shavar_list_info_parse_spec.rb

@@ -0,0 +1,48 @@
+require 'spec_helper'
+
+describe "Faraday Middleware list info parsing" do
+
+  before(:all) do
+    @parser = Faraday::Response::SafeBrowsingUpdateParser.new
+  end
+  
+  it "should parse sample test data one" do
+    resp = <<-END_OF_SAMPLE
+n:1200
+i:googpub-phish-shavar
+u:cache.google.com/first_redirect_example
+u:cache.google.com/second_redirect_example
+sd:1,2
+i:acme-white-shavar
+u:cache.google.com/second_redirect_example
+ad:1-2,4-5,7
+sd:2-6
+    END_OF_SAMPLE
+
+    r = @parser.parse resp
+    r.next.should eq(1200)
+    r.rekey.should be_nil
+    r.reset.should be_nil
+    
+    phish_list = r.get_list('googpub-phish-shavar')
+    phish_list[:u].should eq(["cache.google.com/first_redirect_example", "cache.google.com/second_redirect_example"])
+    phish_list[:sd].should eq([1, 2])
+    phish_list[:ad].should be_empty
+    
+  end
+  
+  it "should stop parsing if reset is in the response" do
+    resp = <<-END_OF_SAMPLE
+n:1200
+r:pleasereset
+i:goog-phish-shavar
+u:cache.google.com/first_redirect_example
+    END_OF_SAMPLE
+  
+    r = @parser.parse resp
+    r.next.should eq(1200)
+    r.reset.should be_true
+    r.has_lists?.should be_false
+  end
+  
+end

data/lib/safe_browsing_task.rb

@@ -0,0 +1,5 @@
+class SafeBrowsingTask < Rails::Railtie
+  rake_tasks do
+    Dir[File.join(File.expand_path('../', __FILE__),'tasks/**/*.rake')].each {|f| load f}
+  end
+end

data/lib/tasks/google.rake

@@ -0,0 +1,122 @@
+require File.expand_path('../../google/safe_browsing_update_helper', __FILE__)
+
+namespace :google do
+  namespace :safe_browsing do
+
+    include Google::SafeBrowsingUpdateHelper
+
+    module GoogleTask
+      extend self
+      
+      def gsb_api
+        @gsb_api ||= Google::SafeBrowsingClient.new
+      end
+
+      def load_remote_shavar_info list
+        update_obj = GoogleTask.gsb_api.shavar_data_update list
+        update_local_shavar_info update_obj
+        update_obj
+      end
+    
+      def load_remote_shavar_chunk url
+        chunk_data = GoogleTask.gsb_api.chunk_data(url)
+        adds, subs = Google::SafeBrowsingParser.parse_shavar_list(chunk_data)
+      end
+    
+      def load_redirect_urls urls, shavar_list
+        Rails.logger.info "Redirect urls: #{urls.join(%Q(\n))}"
+        urls.each do |url|
+          load_redirect_url url, shavar_list
+        end
+      end
+
+      def load_history_redirect_urls urls
+        Rails.logger.info "Inspecting history failed redirect downloads..."
+        Rails.logger.info "Found history failed redirect urls: #{urls.blank?? 'none' : urls.map(&:url).join(%Q(\n))}"
+        urls.each do |url|
+          load_redirect_url url.url, url.list
+        end
+      end
+    
+      def load_redirect_url url, shavar_list
+        begin
+          Rails.logger.info "Loading data for [#{shavar_list.name}] from redirect url #{url}..."
+          adds, subs = load_remote_shavar_chunk(url)
+          update_shavar_chunk adds, subs, shavar_list
+          update_redirect_urls(url, shavar_list,
+                    {download_state: Google::SafeBrowsingRedirectUrl::COMPLETED, last_download_at: Time.now})
+        
+          Rails.logger.info "Finish loading data from #{url}\n"
+        rescue Google::Error::NoContent => e
+          Rails.logger.info "NoContent Error for redirect url [#{url}], continue..."
+          update_redirect_urls url, shavar_list, {download_state: e.message, last_download_at: Time.now}
+        rescue Exception => e
+          Rails.logger.info "Error (#{e.inspect}) for redirect url [#{url}], abort..."
+          update_redirect_urls url, shavar_list, {download_state: e.message, last_download_at: Time.now}
+          raise
+        end
+      end
+
+      def history_redirect_downloads
+        Google::SafeBrowsingRedirectUrl.where("download_state is NULL or download_state != ?", Google::SafeBrowsingRedirectUrl::COMPLETED)
+                                       .order("google_safe_browsing_redirect_urls.order").all
+      end
+      
+    end
+    
+    task :set_logger => :environment do
+      Rails.logger = Logger.new(Rails.root.join('log', "google_safe_browsing_#{Rails.env}.log"), 7, 10240000)
+    end
+    
+    desc "Update GoogleSafeBrowsing data"
+    task :load_remote => :set_logger do
+      Rails.logger = Logger.new(Rails.root.join('log', "google_safe_browsing_#{Rails.env}.log"), 7, 10240000)
+      
+      Rails.logger.info "=== start load_remote at #{Time.now} ==="
+      
+      GoogleTask.load_history_redirect_urls(GoogleTask.history_redirect_downloads)
+      
+      next_updated_at = safe_browsing_service.next_updated_at
+      if !next_updated_at.nil? && next_updated_at > Time.now
+        abort "Google suggests next update until #{next_updated_at}, but now is #{Time.now}"
+      end
+      
+      [Google::SafeBrowsingList::MalwareList, Google::SafeBrowsingList::PhishList].each do |list_name|
+        shavar_list = Google::SafeBrowsingList.find_by_name(list_name.to_s)
+      
+        update_obj = GoogleTask.load_remote_shavar_info list_name
+
+        if (redirect_urls = update_obj.get_redirect_urls(list_name)).empty?
+          Rails.logger.info "No redirect urls detected for #{list_name}"
+        else
+          (new_redirects = []).tap do
+            redirect_urls.each do |url|
+              local_obj = Google::SafeBrowsingRedirectUrl.for_url_and_list_id(url, shavar_list.id).first
+              unless local_obj.nil?
+                Rails.logger.info "Skip the url as it's been processed before #{url}"
+              else
+                new_redirects << url
+              end
+            end
+          end
+          
+          save_redirect_urls(new_redirects, shavar_list)
+          GoogleTask.load_redirect_urls(redirect_urls, shavar_list)
+        end
+
+        Rails.logger.info "=== end of #{list_name} ==="
+      end
+      
+    end
+
+    desc "Seed Google Safe Browsing data"
+    task :db_seed => :set_logger do
+      Google::Function.where(name: Google::Function::GoogleSafeBrowsing).first_or_create
+      
+      [Google::SafeBrowsingList::MalwareList, Google::SafeBrowsingList::PhishList].each do |list|
+        Google::SafeBrowsingList.where(name: list).first_or_create
+      end
+    end
+
+  end
+end

metadata

@@ -0,0 +1,222 @@
+--- !ruby/object:Gem::Specification
+name: google-safe-browsing-plugin
+version: !ruby/object:Gem::Version
+  prerelease: 
+  version: 0.1.1
+platform: ruby
+authors:
+- Lonex
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2013-05-02 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 2.3.0
+    none: false
+  prerelease: false
+  name: activesupport
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 2.3.0
+    none: false
+  type: :development
+- !ruby/object:Gem::Dependency
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '0.8'
+    none: false
+  prerelease: false
+  name: faraday
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '0.8'
+    none: false
+  type: :development
+- !ruby/object:Gem::Dependency
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 2.8.0
+    none: false
+  prerelease: false
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 2.8.0
+    none: false
+  type: :development
+- !ruby/object:Gem::Dependency
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '3.12'
+    none: false
+  prerelease: false
+  name: rdoc
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '3.12'
+    none: false
+  type: :development
+- !ruby/object:Gem::Dependency
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 1.2.0
+    none: false
+  prerelease: false
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 1.2.0
+    none: false
+  type: :development
+- !ruby/object:Gem::Dependency
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 1.8.4
+    none: false
+  prerelease: false
+  name: jeweler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 1.8.4
+    none: false
+  type: :development
+- !ruby/object:Gem::Dependency
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.8.7
+    none: false
+  prerelease: false
+  name: faraday
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.8.7
+    none: false
+  type: :runtime
+- !ruby/object:Gem::Dependency
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 2.3.0
+    none: false
+  prerelease: false
+  name: activesupport
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 2.3.0
+    none: false
+  type: :runtime
+description: A Ruby implementation of the Google Safe Browsing v2. Rails is the dependency
+  mainly because of the data layer.
+email: stonelonely@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files:
+- LICENSE.txt
+- README.md
+files:
+- lib/faraday/response/safe_browsing_update_parser.rb
+- lib/google/safe_browsing_client.rb
+- lib/google/safe_browsing_parser.rb
+- lib/google/safe_browsing_update_helper.rb
+- lib/google/sha_util.rb
+- lib/google/url_canonicalizer.rb
+- lib/google/url_scramble.rb
+- lib/google_safe_browsing_plugin.rb
+- lib/rails/generators/google/config/config_generator.rb
+- lib/rails/generators/google/config/templates/google_safe_browsing.yml
+- lib/rails/generators/google/helper/helper_generator.rb
+- lib/rails/generators/google/helper/templates/safe_browsing_helper.rb
+- lib/rails/generators/google/install_generator.rb
+- lib/rails/generators/google/model/model_generator.rb
+- lib/rails/generators/google/model/templates/create_google_functions.rb
+- lib/rails/generators/google/model/templates/create_google_safe_browsing_full_hash_requests.rb
+- lib/rails/generators/google/model/templates/create_google_safe_browsing_full_hashes.rb
+- lib/rails/generators/google/model/templates/create_google_safe_browsing_list.rb
+- lib/rails/generators/google/model/templates/create_google_safe_browsing_redirect_urls.rb
+- lib/rails/generators/google/model/templates/create_google_safe_browsing_shavar.rb
+- lib/rails/generators/google/model/templates/google.rb
+- lib/rails/generators/google/model/templates/google/error.rb
+- lib/rails/generators/google/model/templates/google/function.rb
+- lib/rails/generators/google/model/templates/google/safe_browsing_full_hash.rb
+- lib/rails/generators/google/model/templates/google/safe_browsing_full_hash_request.rb
+- lib/rails/generators/google/model/templates/google/safe_browsing_list.rb
+- lib/rails/generators/google/model/templates/google/safe_browsing_redirect_url.rb
+- lib/rails/generators/google/model/templates/google/safe_browsing_shavar.rb
+- lib/rails/generators/google/model/templates/google/safe_browsing_update.rb
+- lib/rails/generators/google/rspec/rspec_generator.rb
+- lib/rails/generators/google/rspec/templates/bin_sample_1.data
+- lib/rails/generators/google/rspec/templates/bin_sample_2.data
+- lib/rails/generators/google/rspec/templates/full_hash_parse_spec.rb
+- lib/rails/generators/google/rspec/templates/full_hash_response_0.data
+- lib/rails/generators/google/rspec/templates/full_hash_response_1.data
+- lib/rails/generators/google/rspec/templates/full_hash_response_2.data
+- lib/rails/generators/google/rspec/templates/full_hash_response_3.data
+- lib/rails/generators/google/rspec/templates/shavar_encode_data_parse_spec.rb
+- lib/rails/generators/google/rspec/templates/shavar_list_info_parse_spec.rb
+- lib/safe_browsing_task.rb
+- lib/tasks/google.rake
+- LICENSE.txt
+- README.md
+homepage: http://github.com/lonex/google-safe-browsing-plugin
+licenses:
+- MIT
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+      segments:
+      - 0
+      hash: -3952199818102085151
+  none: false
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+  none: false
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.24
+signing_key: 
+specification_version: 3
+summary: Rails plugin for Google Safe Browsing
+test_files: []