malware_api 0.1

data/MIT-LICENSE

@@ -0,0 +1,20 @@
+Copyright (c) 2009 [name of plugin creator]
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.markdown

@@ -0,0 +1,39 @@
+Malware
+=======
+
+Malware is a rails plugin to take advantage of the Google Safe Browsing API
+
+Installation
+============
+
+Request an API key from http://code.google.com/apis/safebrowsing/key_signup.html
+
+Create a config file with your API key. In `RAILS_ROOT`/config/initializers/malware_key.rb
+
+    # API key for Google Safe Browsing API
+    # http://code.google.com/apis/safebrowsing/
+    SAFEBROWSING_API_KEY = "ABQIAAAAN_MxjBsAgBKC8N1cqNloZhT3elDgi-bgogZuFD30ho7emQ1XEw"
+
+Install the plugin
+
+    gem install malware
+
+Generate and run a migration to create the tables
+
+    script/generate malware_migration
+    rake db:migrate
+
+Example
+=======
+
+    $ script/console 
+    Loading development environment (Rails 2.3.5)
+    >> Malware.update
+    => "Updated malware info (in 73.865188s) +301657/-0 (1.47438 1.18185)\n"
+    >> Malware.check('http://malware.testing.google.test/testing/malware/')
+    => true
+    >> Malware.check('http://www.jorgebernal.info/')
+    => false
+
+
+Copyright (c) 2009 Jorge Bernal <koke@amedias.org>, released under the MIT license

data/Rakefile

@@ -0,0 +1,58 @@
+require 'rake'
+require 'rake/testtask'
+require 'rake/rdoctask'
+require 'rake/gempackagetask'
+
+desc 'Default: run unit tests.'
+task :default => :test
+
+desc 'Test the malware plugin.'
+Rake::TestTask.new(:test) do |t|
+  t.libs << 'lib'
+  t.libs << 'test'
+  t.pattern = 'test/**/*_test.rb'
+  t.verbose = true
+end
+
+desc 'Generate documentation for the malware plugin.'
+Rake::RDocTask.new(:rdoc) do |rdoc|
+  rdoc.rdoc_dir = 'rdoc'
+  rdoc.title    = 'Malware'
+  rdoc.options << '--line-numbers' << '--inline-source'
+  rdoc.rdoc_files.include('README')
+  rdoc.rdoc_files.include('lib/**/*.rb')
+end
+
+namespace :db do
+  namespace :migrate do
+    description = "Migrate the database through scripts in vendor/plugins/malware/lib/db/migrate"
+    description << "and update db/schema.rb by invoking db:schema:dump."
+    description << "Target specific version with VERSION=x. Turn off output with VERBOSE=false."
+    
+    desc description
+    task :malware => :environment do
+      ActiveRecord::Migration.verbose = ENV["VERBOSE"] ? ENV["VERBOSE"] == "true" : true
+      ActiveRecord::Migrator.migrate("vendor/plugins/malware/lib/db/migrate/", ENV["VERSION"] ? ENV["VERSION"].to_i : nil)
+      Rake::Task["db:schema:dump"].invoke if ActiveRecord::Base.schema_format == :ruby
+    end
+  end
+end 
+
+desc %{Update ".manifest" with the latest list of project filenames. Respect\
+.gitignore by excluding everything that git ignores. Update `files` and\
+`test_files` arrays in "*.gemspec" file if it's present.}
+task :manifest do
+  list = `git ls-files --full-name --exclude=*.gemspec --exclude=.*`.chomp.split("\n")
+  
+  if spec_file = Dir['*.gemspec'].first
+    spec = File.read spec_file
+    spec.gsub! /^(\s* s.(test_)?files \s* = \s* )( \[ [^\]]* \] | %w\( [^)]* \) )/mx do
+      assignment = $1
+      bunch = $2 ? list.grep(/^test\//) : list
+      '%s%%w(%s)' % [assignment, bunch.join(' ')]
+    end
+      
+    File.open(spec_file, 'w') { |f| f << spec }
+  end
+  File.open('.manifest', 'w') { |f| f << list.join("\n") }
+end

data/generators/malware_migration/malware_migration_generator.rb

@@ -0,0 +1,9 @@
+class MalwareMigrationGenerator < Rails::Generator::Base
+  def manifest
+    record do |m|
+      m.migration_template 'migration.rb', "db/migrate", {
+        :migration_file_name => "create_malware_tables"  
+      }
+    end
+  end
+end

data/generators/malware_migration/templates/migration.rb

@@ -0,0 +1,21 @@
+class CreateMalwareTables < ActiveRecord::Migration
+  def self.up
+    create_table :malwares do |t|
+      t.integer :black_major
+      t.integer :black_minor
+      t.integer :malware_major
+      t.integer :malware_minor
+
+      t.timestamps
+    end
+
+    create_table :malware_hashes do |t|
+      t.string :url, :limit => 32, :null => false
+    end
+  end
+
+  def self.down
+    drop_table :malware_hashes
+    drop_table :malwares
+  end
+end

data/install.rb

@@ -0,0 +1 @@
+# Install hook code here

data/lib/app/models/malware.rb

@@ -0,0 +1,111 @@
+require "open-uri"
+
+class Malware < ActiveRecord::Base
+  before_create :delete_previous
+  
+  # Inserting/deleting one entry at a time is painfully slow
+  # Inserting in batches of 100 entries reduces time to ~1 minute
+  @batch_size = 100
+  
+  # Fetch changes from google servers and store them in the database
+  # Might be a slow operation, specially on the first run
+  def self.update
+    options = find(:first, :order => "created_at DESC")
+    unless options
+      options = self.new
+      options.black_major ||= 1
+      options.black_minor ||= -1
+      options.malware_major ||= 1
+      options.malware_minor ||= -1
+      
+      options.save
+    end
+    
+    update_uri = "http://sb.google.com/safebrowsing/update?client=api" +
+                 "&apikey=#{SAFEBROWSING_API_KEY}" +
+                 "&version=goog-black-hash:#{options.black_major}:#{options.black_minor}," +
+                 "goog-malware-hash:#{options.malware_major}:#{options.malware_minor}"
+
+    logger.info("Updating malware info (#{options.black_major}.#{options.black_minor} #{options.malware_major}.#{options.malware_minor})")
+    added = 0
+    removed = 0
+    to_add = []
+    to_remove = []
+    started = Time.now
+    
+    # FIXME: try batch inserts if performance becomes an issue
+    open(update_uri) do |f|
+      f.each_line do |line|
+        if (line =~ /\[goog-black-hash (\d+).(\d+)/)
+          options.black_major = $1
+          options.black_minor = $2
+          logger.info("Found new black version (#{options.black_major},#{options.black_minor})")
+        elsif (line =~ /\[goog-malware-hash (\d+).(\d+)/)
+          options.malware_major = $1
+          options.malware_minor = $2
+          logger.info("Found new malware version (#{options.malware_major}.#{options.malware_minor})")
+        elsif (line =~ /\+([0-9a-f]{32})/)
+          to_add << $1
+          # MalwareHash.create(:url => $1)
+          added += 1
+        elsif (line =~ /-([0-9a-f]{32})/)
+          to_remove << $1
+          # MalwareHash.delete_all(["url = ?", $1])
+          removed += 1
+        end
+        
+        if to_add.size > @batch_size
+          MalwareHash.add_batch(to_add)
+          to_add = []
+        end
+        
+        if to_remove.size > @batch_size
+          MalwareHash.delete_batch(to_remove)
+          to_remove = []
+        end
+        
+      end
+    end
+
+    MalwareHash.add_batch(to_add)
+    MalwareHash.delete_batch(to_remove)
+    
+    options.save
+    logger.info("Updated malware info (in #{Time.now - started}s) +#{added}/-#{removed} (#{options.black_major}.#{options.black_minor} #{options.malware_major}.#{options.malware_minor})")
+  end
+  
+  # Check a url against the malware database
+  # URLs can be in any form:
+  # * http://example.com/foo/bar.html
+  # * example.com/foo/
+  # * example.com
+  def self.check(url)
+    # Make sure url is according to http://code.google.com/apis/safebrowsing/developers_guide.html#ListFormat
+    url.gsub!(/^http:\/\//, '')
+    url.downcase!
+    alt_url = nil
+    
+    if url =~ /\//
+      # Contains a /
+      url =~ /^([^\/]+\/)/
+      alt_url = $1
+    else
+      # Doesn't contain a slash -> hostname
+      url += "/"
+    end
+    
+    if MalwareHash.check(url)
+      return true
+    elsif alt_url
+      return MalwareHash.check(alt_url)
+    else
+      return false
+    end
+  end
+  
+private
+
+  def delete_previous
+    Malware.delete_all
+  end
+end

data/lib/app/models/malware_hash.rb

@@ -0,0 +1,22 @@
+class MalwareHash < ActiveRecord::Base
+  validates_uniqueness_of :url
+  validates_length_of :url, :is => 32
+  
+  def self.check(url)
+    not find(:first, :conditions => ["url = MD5( ? )", url]).nil?
+  end
+  
+  def self.add_batch(urls)
+    return if urls.empty?
+    
+    values = urls.map {|url| sanitize_sql(["(?)", url])}.join(',')
+    MalwareHash.connection.execute("INSERT INTO #{MalwareHash.table_name} (url) VALUES #{values}")
+  end
+  
+  def self.delete_batch(urls)
+    return if urls.empty?
+
+    values = urls.map {|url| sanitize_sql(["?", url])}.join(',')
+    MalwareHash.connection.execute("DELETE FROM #{MalwareHash.table_name} WHERE url IN (#{values})")
+  end
+end

data/lib/db/migrate/20091225014508_create_malwares.rb

@@ -0,0 +1,16 @@
+class CreateMalwares < ActiveRecord::Migration
+  def self.up
+    create_table :malwares do |t|
+      t.integer :black_major
+      t.integer :black_minor
+      t.integer :malware_major
+      t.integer :malware_minor
+
+      t.timestamps
+    end
+  end
+
+  def self.down
+    drop_table :malwares
+  end
+end

data/lib/db/migrate/20091225014608_create_malware_hashes.rb

@@ -0,0 +1,11 @@
+class CreateMalwareHashes < ActiveRecord::Migration
+  def self.up
+    create_table :malware_hashes do |t|
+      t.string :url, :limit => 32, :null => false
+    end
+  end
+
+  def self.down
+    drop_table :malware_hashes
+  end
+end

data/lib/malware_api.rb

@@ -0,0 +1,8 @@
+# Malware
+
+%w{ models }.each do |dir|
+  path = File.join(File.dirname(__FILE__), 'app', dir)
+  $LOAD_PATH << path
+  ActiveSupport::Dependencies.load_paths << path
+  ActiveSupport::Dependencies.load_once_paths.delete(path)
+end 

data/rails/init.rb

@@ -0,0 +1 @@
+require "malware_api"

data/tasks/malware_tasks.rake

@@ -0,0 +1,14 @@
+# namespace :db do
+#   namespace :migrate do
+#     description = "Migrate the database through scripts in vendor/plugins/malware/lib/db/migrate"
+#     description << "and update db/schema.rb by invoking db:schema:dump."
+#     description << "Target specific version with VERSION=x. Turn off output with VERBOSE=false."
+#     
+#     desc description 
+#     task :malware => :environment do
+#       ActiveRecord::Migration.verbose = ENV["VERBOSE"] ? ENV["VERBOSE"] == "true" : true
+#       ActiveRecord::Migrator.migrate("vendor/plugins/malware/lib/db/migrate/", ENV["VERSION"] ? ENV["VERSION"].to_i : nil)
+#       Rake::Task["db:schema:dump"].invoke if ActiveRecord::Base.schema_format == :ruby
+#     end
+#   end
+# end

data/test/database.yml

@@ -0,0 +1,6 @@
+mysql:
+  :adapter: mysql
+  :host: localhost
+  :username: root
+  :password: 
+  :database: malware_plugin_test

data/test/malware_test.rb

@@ -0,0 +1,10 @@
+require 'test_helper'
+
+class MalwareTest < ActiveSupport::TestCase
+  load_schema
+  
+  test "loaded ok" do
+    assert_kind_of(Malware, Malware.new)
+    assert_kind_of(MalwareHash, MalwareHash.new)
+  end  
+end

data/test/schema.rb

@@ -0,0 +1,14 @@
+ActiveRecord::Schema.define(:version => 0) do
+  create_table "malware_hashes", :force => true do |t|
+    t.string "url", :limit => 32, :null => false
+  end
+
+  create_table "malwares", :force => true do |t|
+    t.integer  "black_major"
+    t.integer  "black_minor"
+    t.integer  "malware_major"
+    t.integer  "malware_minor"
+
+    t.timestamps
+  end
+end 

data/test/test_helper.rb

@@ -0,0 +1,20 @@
+require 'rubygems'
+require 'active_support'
+require 'active_support/test_case'
+
+ENV['RAILS_ENV'] = 'test' 
+ENV['RAILS_ROOT'] ||= File.dirname(__FILE__) + '/../../../..'
+
+require 'test/unit'
+require File.expand_path(File.join(ENV['RAILS_ROOT'], 'config/environment.rb')) 
+
+def load_schema
+  config = YAML::load(IO.read(File.dirname(__FILE__) + '/database.yml')) 
+  
+  ActiveRecord::Base.logger = Logger.new(File.dirname(__FILE__) + "/debug.log")
+  db_adapter = ENV['DB'] || "mysql"
+  
+  ActiveRecord::Base.establish_connection(config[db_adapter])
+  load(File.dirname(__FILE__) + "/schema.rb")
+  require File.dirname(__FILE__) + '/../rails/init.rb'
+end

data/uninstall.rb

@@ -0,0 +1 @@
+# Uninstall hook code here

metadata

@@ -0,0 +1,76 @@
+--- !ruby/object:Gem::Specification 
+name: malware_api
+version: !ruby/object:Gem::Version 
+  version: "0.1"
+platform: ruby
+authors: 
+- Jorge Bernal
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2009-12-26 00:00:00 +01:00
+default_executable: 
+dependencies: []
+
+description: 
+email: koke@amedias.org
+executables: []
+
+extensions: []
+
+extra_rdoc_files: 
+- README.markdown
+files: 
+- MIT-LICENSE
+- README.markdown
+- Rakefile
+- generators/malware_migration/USAGE
+- generators/malware_migration/malware_migration_generator.rb
+- generators/malware_migration/templates/migration.rb
+- install.rb
+- lib/app/models/malware.rb
+- lib/app/models/malware_hash.rb
+- lib/db/migrate/20091225014508_create_malwares.rb
+- lib/db/migrate/20091225014608_create_malware_hashes.rb
+- lib/malware_api.rb
+- rails/init.rb
+- tasks/malware_tasks.rake
+- test/database.yml
+- test/malware_test.rb
+- test/schema.rb
+- test/test_helper.rb
+- uninstall.rb
+has_rdoc: true
+homepage: http://github.com/koke/malware_api
+licenses: []
+
+post_install_message: 
+rdoc_options: []
+
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 1.3.5
+signing_key: 
+specification_version: 3
+summary: Google Safe Browsing API for Rails
+test_files: 
+- test/database.yml
+- test/malware_test.rb
+- test/schema.rb
+- test/test_helper.rb