google-safe-browsing-plugin 0.1.1

data/lib/rails/generators/google/install_generator.rb

@@ -0,0 +1,20 @@
+module Google
+  module Generators
+    class InstallGenerator < Rails::Generators::Base
+      
+      GENERATORS = %w(google:config google:helper google:model google:rspec)
+      def run_all_generators
+        if behavior == :invoke
+          GENERATORS.each do |g|
+            generate g
+          end
+        elsif behavior == :revoke
+          GENERATORS.reverse.each do |g|
+            Rails::Generators.invoke g, [], :behavior => :revoke
+          end
+        end
+      end
+      
+    end
+  end
+end

data/lib/rails/generators/google/model/model_generator.rb

@@ -0,0 +1,47 @@
+require 'rails/generators'
+require 'rails/generators/migration'
+require 'rails/generators/active_record'
+
+module Google
+  module Generators
+    class ModelGenerator < Rails::Generators::Base
+      include Rails::Generators::Migration
+      
+      source_root File.expand_path('../templates', __FILE__)
+  
+      def self.next_migration_number path
+        ActiveRecord::Generators::Base.next_migration_number(path)
+      end
+
+      def creaet_migrations
+        
+        %w(create_google_functions.rb
+           create_google_safe_browsing_full_hash_requests.rb 
+           create_google_safe_browsing_list.rb
+           create_google_safe_browsing_shavar.rb
+           create_google_safe_browsing_full_hashes.rb
+           create_google_safe_browsing_redirect_urls.rb).each do |f|
+        
+          migration_template "#{f}", "db/migrate/#{f}"
+        end
+
+      end
+
+      def create_models
+        %w(google.rb
+           google/function.rb
+           google/error.rb 
+           google/safe_browsing_full_hash.rb
+           google/safe_browsing_full_hash_request.rb
+           google/safe_browsing_list.rb
+           google/safe_browsing_redirect_url.rb
+           google/safe_browsing_shavar.rb
+           google/safe_browsing_update.rb
+        ).each do |f|
+          template "#{f}", "app/models/#{f}"
+        end
+      end
+
+    end
+  end
+end

data/lib/rails/generators/google/model/templates/create_google_functions.rb

@@ -0,0 +1,18 @@
+class CreateGoogleFunctions < ActiveRecord::Migration
+
+  class << self
+    def up
+      create_table :google_functions do |t|
+        t.string :name, :limit => 255
+        t.integer :version
+        t.timestamp :next_updated_at
+        t.timestamps
+      end
+    end
+  
+    def down
+      drop_table :google_functions
+    end
+    
+  end
+end

data/lib/rails/generators/google/model/templates/create_google_safe_browsing_full_hash_requests.rb

@@ -0,0 +1,22 @@
+class CreateGoogleSafeBrowsingFullHashRequests < ActiveRecord::Migration
+  
+  class << self
+    def up
+      create_table :google_safe_browsing_full_hash_requests do |t|
+        t.string :prefix
+        t.string :state
+        t.integer :attempts
+        t.datetime :created_at
+        t.datetime :requested_at
+      end
+    
+      add_index :google_safe_browsing_full_hash_requests, :prefix, :name => 'index_hash_prefix'
+    
+    end
+  
+    def down
+      drop_table :google_safe_browsing_full_hash_requests
+    end
+
+  end
+end

data/lib/rails/generators/google/model/templates/create_google_safe_browsing_full_hashes.rb

@@ -0,0 +1,20 @@
+class CreateGoogleSafeBrowsingFullHashes < ActiveRecord::Migration
+
+  class << self
+    def up
+      create_table :google_safe_browsing_full_hashes do |t|
+        t.string :value   # 32 Bytes
+        t.integer :add_chunk_num
+        t.integer :google_safe_browsing_list_id
+        t.timestamps
+      end
+    
+      add_index :google_safe_browsing_full_hashes, :value, :name => 'index_full_hashes'
+    
+    end
+
+    def down
+      drop_table :google_safe_browsing_full_hashes
+    end
+  end
+end

data/lib/rails/generators/google/model/templates/create_google_safe_browsing_list.rb

@@ -0,0 +1,15 @@
+class CreateGoogleSafeBrowsingList < ActiveRecord::Migration
+
+  class << self
+    def up
+      create_table :google_safe_browsing_lists do |t|
+        t.string :name
+      end
+    end
+  
+    def down
+      drop_table :google_safe_browsing_lists
+    end
+  end
+    
+end

data/lib/rails/generators/google/model/templates/create_google_safe_browsing_redirect_urls.rb

@@ -0,0 +1,26 @@
+class CreateGoogleSafeBrowsingRedirectUrls < ActiveRecord::Migration
+  
+  class << self
+    def up
+      create_table :google_safe_browsing_redirect_urls do |t|
+        t.string :url, :limit => 2047
+        t.string :url_hash        
+        t.integer :order
+        t.integer :google_safe_browsing_list_id
+        t.string :download_state
+        t.integer :download_attempts
+        t.datetime :last_download_at
+        t.timestamps
+      end
+    
+      add_index :google_safe_browsing_redirect_urls, :url_hash, :name => 'index_redirect_url_hashes'
+      add_index :google_safe_browsing_redirect_urls, :order, :name => 'index_redirect_urls_order'
+    
+    end
+  
+    def down
+      drop_table :google_safe_browsing_redirect_urls
+    end
+    
+  end
+end

data/lib/rails/generators/google/model/templates/create_google_safe_browsing_shavar.rb

@@ -0,0 +1,27 @@
+class CreateGoogleSafeBrowsingShavar < ActiveRecord::Migration
+  
+  class << self
+    def up
+      create_table :google_safe_browsing_shavars do |t|
+        t.integer :chunk_num
+        t.string :chunk_type, :limit => 1  # "a" or "s"
+        t.string :host_key
+        t.integer :add_chunk_num  # Only for sub shavar data, add shavar data will have it as NULL
+        t.string :prefix
+        t.integer :google_safe_browsing_list_id  # malware or phishing
+      end
+
+      add_index :google_safe_browsing_shavars, [:chunk_type, :host_key], :name => 'index_chunk_type_host'
+      add_index :google_safe_browsing_shavars, [:chunk_type, :host_key, :prefix], :name => 'index_chunk_type_host_prefix'
+      add_index :google_safe_browsing_shavars, [:chunk_type, :add_chunk_num, :host_key, :prefix], :name => 'index_add_chunk_host_prefix'
+    
+      add_index :google_safe_browsing_shavars, \
+                [:google_safe_browsing_list_id, :chunk_type, :chunk_num, :host_key, :add_chunk_num, :prefix], :unique => true, :name => 'index_chunk_host_prefix'
+    end
+
+    def down
+      drop_table :google_safe_browsing_shavars
+    end
+  end
+  
+end

data/lib/rails/generators/google/model/templates/google.rb

@@ -0,0 +1,2 @@
+module Google
+end

data/lib/rails/generators/google/model/templates/google/error.rb

@@ -0,0 +1,11 @@
+module Google
+  module Error
+
+    class ParserError < StandardError; end
+    class InvalidRequest < StandardError; end
+    class ServiceUnavailable < StandardError; end
+    class NoContent < StandardError; end
+    
+    class UnknownError < StandardError; end
+  end
+end

data/lib/rails/generators/google/model/templates/google/function.rb

@@ -0,0 +1,6 @@
+module Google
+  class Function < ActiveRecord::Base
+    GoogleSafeBrowsing ||= 'GoogleSafeBrowsing'
+    attr_accessible :name, :version, :next_updated_at
+  end
+end

data/lib/rails/generators/google/model/templates/google/safe_browsing_full_hash.rb

@@ -0,0 +1,7 @@
+module Google
+  class SafeBrowsingFullHash < ActiveRecord::Base
+    belongs_to :list, :class_name => "Google::SafeBrowsingList", :foreign_key => "google_safe_browsing_list_id"
+  
+    attr_accessible :add_chunk_num, :value, :google_safe_browsing_list_id
+  end  
+end

data/lib/rails/generators/google/model/templates/google/safe_browsing_full_hash_request.rb

@@ -0,0 +1,19 @@
+module Google
+  class SafeBrowsingFullHashRequest < ActiveRecord::Base
+    before_save :set_other_attrs
+    
+    attr_accessible :prefix, :state, :attempts, :requested_at
+    
+    COMPLETED ||= 'completed'
+    
+    def set_other_attrs
+      if !self.state.blank? && self.state != COMPLETED
+        self.attempts ||= 0
+        self.attempts += 1
+      else
+        self.attempts = nil
+      end
+    end
+    
+  end  
+end

data/lib/rails/generators/google/model/templates/google/safe_browsing_list.rb

@@ -0,0 +1,41 @@
+module Google
+  class SafeBrowsingList < ActiveRecord::Base
+    
+    MalwareList ||= 'goog-malware-shavar'
+    PhishList   ||= 'googpub-phish-shavar'
+  
+    attr_accessible :prefix
+ 
+    has_many :shavars, :class_name => "Google::SafeBrowsingShavar", :foreign_key => "google_safe_browsing_list_id", :dependent => :destroy
+    has_many :redirect_urls, :class_name => "Google::SafeBrowsingRedirectUrl", :foreign_key => "google_safe_browsing_list_id", :dependent => :destroy
+    has_many :full_hashes, :class_name => "Google::SafeBrowsingFullHash", :foreign_key => "google_safe_browsing_list_id", :dependent => :destroy
+
+    class << self
+      
+      def valid_list? list_name
+        [MalwareList, PhishList].include?(list_name.to_s)
+      end
+      
+      def malware_list
+        @malware_list_obj ||= find_by_name MalwareList
+      end
+      
+      def phishing_list
+        @phishing_list_obj ||= find_by_name PhishList
+      end
+      
+      def list_by_name name
+        if valid_list?(name.to_s)
+          if malware_list.name == name.to_s
+            malware_list
+          elsif phishing_list.name == name.to_s
+            phishing_list
+          end
+        else
+          nil
+        end
+      end
+      
+    end
+  end
+end

data/lib/rails/generators/google/model/templates/google/safe_browsing_redirect_url.rb

@@ -0,0 +1,36 @@
+module Google
+  class SafeBrowsingRedirectUrl < ActiveRecord::Base
+    
+    belongs_to :list, :class_name => "Google::SafeBrowsingList", :foreign_key => "google_safe_browsing_list_id"
+    attr_accessible :url, :order, :download_state, :last_download_at, :google_safe_browsing_list_id
+    before_create :set_other_attrs
+    before_update :set_download_attr
+    
+    COMPLETED ||= 'completed'
+    
+    scope :for_url_and_list_id, lambda { |url, list_id|
+      where(url_hash: SafeBrowsingRedirectUrl.url_hash_key(url), google_safe_browsing_list_id: list_id)
+    } 
+
+    def set_other_attrs
+      ord = Google::SafeBrowsingRedirectUrl.maximum(:order)
+      self.order = ord.nil?? 1 : ord + 1
+      self.url_hash = SafeBrowsingRedirectUrl.url_hash_key(self.url)
+      set_download_attr
+    end
+
+    def set_download_attr
+      if !self.download_state.blank? && self.download_state != COMPLETED
+        self.download_attempts ||= 0
+        self.download_attempts += 1
+      end
+    end
+    
+    class << self
+      def url_hash_key url_str
+        Digest::MD5.hexdigest(url_str)
+      end
+    end
+    
+  end  
+end

data/lib/rails/generators/google/model/templates/google/safe_browsing_shavar.rb

@@ -0,0 +1,38 @@
+module Google
+  class SafeBrowsingShavar < ActiveRecord::Base
+    
+    CHUNK_TYPE_ADD ||= 'a'
+    CHUNK_TYPE_SUB ||= 's'
+  
+    belongs_to :list, :class_name => "Google::SafeBrowsingList", :foreign_key => "google_safe_browsing_list_id"
+  
+    scope :add_chunk_nums_for_list, lambda {|list_name|
+      joins("join google_safe_browsing_lists as lists on lists.id = google_safe_browsing_shavars.google_safe_browsing_list_id")
+         .where("lists.name = ?", list_name)
+         .where('google_safe_browsing_shavars.chunk_type = ?', CHUNK_TYPE_ADD)
+         .group("google_safe_browsing_shavars.chunk_num")
+         .order('google_safe_browsing_shavars.chunk_num')
+    }
+
+    scope :sub_chunk_nums_for_list, lambda {|list_name|
+      joins("join google_safe_browsing_lists as lists on lists.id = google_safe_browsing_shavars.google_safe_browsing_list_id")
+         .where("lists.name = ?", list_name)
+         .where('google_safe_browsing_shavars.chunk_type = ?', CHUNK_TYPE_SUB)
+         .group("google_safe_browsing_shavars.chunk_num")
+         .order('google_safe_browsing_shavars.chunk_num')
+    }
+
+    scope :add_host_keys, lambda { |hashes|
+      where(chunk_type: CHUNK_TYPE_ADD, host_key: hashes)
+    }
+    
+    scope :add_host_prefixes, lambda { |hosts, prefixes|
+      where(chunk_type: CHUNK_TYPE_ADD, host_key: hosts, prefix: prefixes)
+    }
+
+    def self.find_subs_for_add add_chunk_num, host_key, prefix
+      where(chunk_type: CHUNK_TYPE_SUB, add_chunk_num: add_chunk_num, host_key: host_key, prefix: prefix)
+    end
+    
+  end  
+end

data/lib/rails/generators/google/model/templates/google/safe_browsing_update.rb

@@ -0,0 +1,77 @@
+# To capture the GSB Update payload
+module Google
+  class SafeBrowsingUpdate
+    
+    # 
+    # BODY      = [(REKEY | MAC) LF] NEXT LF (RESET | (LIST LF)+) EOF
+    # NEXT      = "n:" DIGIT+                               # Minimum delay before polling again in seconds
+    # REKEY     = "e:pleaserekey"
+    # RESET     = "r:pleasereset"
+    # LIST      = "i:" LISTNAME [MAC] (LF LISTDATA)+
+    # LISTNAME  = (LOALPHA | DIGIT | "-")+                  # e.g. "goog-phish-sha128"
+    # MAC       = "," (LOALPHA | DIGIT)+
+    # LISTDATA  = ((REDIRECT_URL | ADDDEL-HEAD | SUBDEL-HEAD) LF)+
+    # REDIRECT_URL = "u:" URL [MAC]
+    # URL       = Defined in RFC 1738
+    # ADDDEL-HEAD  = "ad:" CHUNKLIST
+    # SUBDEL-HEAD  = "sd:" CHUNKLIST
+    # CHUNKLIST = (RANGE | NUMBER) ["," CHUNKLIST]
+    # NUMBER    = DIGIT+                                    # Chunk number >= 1
+    # RANGE     = NUMBER "-" NUMBER
+    # 
+    
+    attr_accessor :next, :rekey, :reset
+    attr_reader :lists
+    attr_accessor :current_list
+    
+    def set_current_list list_name
+      @current_list = list_name.to_s.to_sym
+      @lists ||= {}
+      # :u is download urls
+      # :sd is sub del
+      # :ad is add del
+      @lists[current_list] ||= { :u => [], :sd => [], :ad => [] }
+    end
+    
+    def has_lists?
+      @lists != nil
+    end
+    
+    def get_list list_name
+      @lists[list_name.to_s.to_sym]
+    end
+    
+    def get_current_list
+      @lists[current_list]
+    end
+    
+    def get_redirect_urls list_name
+      name = list_name.to_s.to_sym
+      if @lists && @lists[name] && !@lists[name][:u].blank?
+        @lists[name][:u]
+      else
+        []
+      end
+    end
+    
+    def get_ad_chunk_ids list_name
+      name = list_name.to_s.to_sym
+      if @lists && @lists[name] && !@lists[name][:ad].blank?
+        @lists[name][:ad]
+      else
+        []
+      end
+    end
+    
+    def get_sd_chunk_ids list_name
+      name = list_name.to_s.to_sym
+      if @lists && @lists[name] && !@lists[name][:sd].blank?
+        @lists[name][:sd]
+      else
+        []
+      end
+    end
+   
+    
+  end
+end