google-safe-browsing-lookup 0.1.0

data/Gemfile

@@ -0,0 +1,13 @@
+source "http://rubygems.org"
+# Add dependencies required to use your google-safe-browsing-lookup.
+
+gem "url", ">= 0"
+
+
+# Dependencies to develop google-safe-browsing-lookup.
+group :development do
+  gem "shoulda", ">= 0"
+  gem "bundler", "~> 1.0.0"
+  gem "jeweler", "~> 1.6.4"
+  gem "rcov", ">= 0"
+end

data/LICENSE.txt

@@ -0,0 +1,20 @@
+Copyright (c) 2011 Julien Sobrier
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.rdoc

@@ -0,0 +1,36 @@
+= google-safe-browsing-lookup
+
+Google Safe Browsing Lookup library for Ruby.
+
+If you need to check less than 10,000 URLs a day against the Google Safe Browsing v2 API (http://code.google.com/apis/safebrowsing/), you can use the Lookup API (http://code.google.com/apis/safebrowsing/lookup_guide.html) as a lighter alternative to the more complex API (http://code.google.com/apis/safebrowsing/developers_guide_v2.html).
+
+You need to get an API key from Google at http://code.google.com/apis/safebrowsing/key_signup.html
+
+== Build and install google-safe-browsing-lookup
+
+    git clone https://github.com/juliensobrier/google-safe-browsing-lookup-ruby
+    rake build
+    rake test
+    rake install
+
+Or
+
+    gem install google-safe-browsing-lookup
+
+
+
+== Contributing to google-safe-browsing-lookup
+ 
+* Check out the latest master to make sure the feature hasn't been implemented or the bug hasn't been fixed yet
+* Check out the issue tracker to make sure someone already hasn't requested it and/or contributed it
+* Fork the project
+* Start a feature/bugfix branch
+* Commit and push until you are happy with your contribution
+* Make sure to add tests for it. This is important so I don't break it in a future version unintentionally.
+* Please try not to mess with the Rakefile, version, or history. If you want to have your own version, or is otherwise necessary, that is fine, but please isolate to its own commit so I can cherry-pick around it.
+
+== Copyright
+
+Copyright (c) 2011 Julien Sobrier. See LICENSE.txt for
+further details.
+

data/Rakefile

@@ -0,0 +1,54 @@
+# encoding: utf-8
+
+require 'rubygems'
+require 'bundler'
+begin
+  Bundler.setup(:default, :development)
+rescue Bundler::BundlerError => e
+  $stderr.puts e.message
+  $stderr.puts "Run `bundle install` to install missing gems"
+  exit e.status_code
+end
+require 'rake'
+
+require 'jeweler'
+Jeweler::Tasks.new do |gem|
+  # gem is a Gem::Specification... see http://docs.rubygems.org/read/chapter/20 for more options
+  gem.name = "google-safe-browsing-lookup"
+  gem.homepage = "http://github.com/juliensobrier/google-safe-browsing-lookup"
+  gem.license = "MIT"
+  gem.summary = "Ruby library for the Google Safe Browsing v2 Lookup API"
+  gem.description = "Google Safe Browsing v2 Lookup is a lighter alternative to the full API. The service allows users to check up to 10,000 URLs a day against their list og phishing and malicious sites. A free API key is required."
+  gem.email = "julien@sobrier.net"
+  gem.authors = ["Julien Sobrier"]
+  # dependencies defined in Gemfile
+end
+Jeweler::RubygemsDotOrgTasks.new
+
+require 'rake/testtask'
+Rake::TestTask.new(:test) do |test|
+  test.libs << 'lib' << 'test'
+  test.pattern = 'test/**/test_*.rb'
+  test.verbose = true
+end
+
+require 'rcov/rcovtask'
+Rcov::RcovTask.new do |test|
+  test.libs << 'test'
+  test.pattern = 'test/**/test_*.rb'
+  test.verbose = true
+  test.rcov_opts << '--exclude "gems/*"'
+end
+
+task :default => :test
+
+# require 'rake/rdoctask'
+require 'rdoc/task'
+Rake::RDocTask.new do |rdoc|
+  version = File.exist?('VERSION') ? File.read('VERSION') : ""
+
+  rdoc.rdoc_dir = 'rdoc'
+  rdoc.title = "google-safe-browsing-lookup #{version}"
+  rdoc.rdoc_files.include('README*')
+  rdoc.rdoc_files.include('lib/**/*.rb')
+end

data/VERSION

@@ -0,0 +1 @@
+0.1.0

data/lib/google-safe-browsing-lookup.rb

@@ -0,0 +1,171 @@
+# The library requires an API key from Google.
+# Sign up for a free key a http://code.google.com/apis/safebrowsing/key_signup.html
+#
+# See README.rdoc for more information about Google Safe Browsing v2 API
+# and this library.
+#
+# Author:: Julien Sobrier (mailto:jsobrier@browshot.com)
+# Copyright:: Copyright (c) 2011 Julien Sobrier
+# License:: Distributes under the same terms as Ruby
+
+require 'uri'
+require 'url'
+require 'net/https'
+
+class SafeBrowsingLookup
+	# API key
+	attr_reader :key
+	# Enable debug & error output to the standard output
+	attr_reader :debug
+	# Enable error output to the standard output
+	attr_reader :error
+	# Contain last error
+	attr_reader :last_error
+	# Library version
+	attr_reader :version
+	# Google API version
+	attr_reader :api_version
+
+	# New client
+	#
+	# +key+:: API key
+	# +debug+:: Set to true to print debug & error output to the standard output. false (disabled) by default.
+	# +debug+:: Set to true to print error output to the standard output. false (disabled) by default.
+	def initialize(key='', debug=false, error=false)
+		@key = key || ''
+		@debug = debug || false
+		@error = error || false
+		@last_error = ''
+
+		@version = '0.1'
+		@api_version = '3.0'
+		
+
+		raise ArgumentError, "Missing API key" if (@key == '')
+	end
+
+	# Lookup a list URLs against the Google Safe Browsing v2 lists.
+	#
+	# Returns a hash <url>: <Gooogle match>. The possible list of values for <Gooogle match> are: "ok" (no match), "malware", "phishing", "malware,phishing" (match both lists) and "error".
+	# 
+	# +urls+:: List of URLs to lookup. The Lookup API allows only 10,000 URL checks a day. if you need more, find a Ruby implementation of the full Google Safe brosing v2 API. Each requests must contain 500 URLs at most. The lookup() method will split the list of URLS in blocks of 500 URLs if needed.
+	def lookup(urls='')
+		if (urls.respond_to?('each') == false)
+			urls = Array.new(1, urls)
+		end
+# 		urls_copy = Array.new(urls)
+
+		results = { }
+
+# 		while (urls_copy.length > 0)
+# 			inputs = urls_copy.slice!(0, 500)
+		count = 0
+		while (count * 500 < urls.length)
+			inputs = urls.slice(count * 500, 500)
+			body = inputs.length.to_s
+			inputs.each do |url|
+				body = body + "\n" + canonical(url)
+			end
+
+			debug("BODY:\n#{body}\n\n");
+			uri = URI.parse("https://sb-ssl.google.com/safebrowsing/api/lookup?client=ruby&apikey=#{@key}&appver=#{@version}&pver=#{@api_version}")
+			
+			http = Net::HTTP.new(uri.host, uri.port)
+			http.open_timeout = 30
+			http.read_timeout = 30
+			http.use_ssl = true
+			http.verify_mode = OpenSSL::SSL::VERIFY_NONE
+
+			response = http.request_post("#{uri.path}?#{uri.query}", body)
+
+			case response
+				when Net::HTTPOK # 200
+					debug("At least 1 match\n")
+					results.merge!( parse(inputs, response.body) )
+
+				when Net::HTTPNoContent # 204
+					debug("No match\n")
+					results.merge!( ok(inputs) )
+
+				when Net::HTTPBadRequest # 400
+					error("Invalid request")
+					results.merge( errors(inputs) )
+
+				when Net::HTTPUnauthorized # 401
+					error("Invalid API key")
+					results.merge!( errors(inputs) )
+
+				when Net::HTTPServiceUnavailable # 503
+					error("Server error, client may have sent too many requests")
+					results.merge!( errors(inputs) )
+
+				else
+					self.error("Unexpected server response: #{response.code}")
+					results.merge!( errors(inputs) )
+			end
+
+		count = count + 1
+		end
+
+		return results
+	end
+
+
+	private
+
+	# Not much is actually done, full URL canonicalization is not required with the Lookup library according to the API documentation
+	def canonical(url='')
+		# remove leading/ending white spaces
+		url.strip!
+
+		# make sure whe have a scheme
+		if (url !~ /^https?\:\/\//i)
+			url = "http://#{url}"
+		end
+
+		uri = URL.new(url)
+		
+		return uri.to_s
+	end
+
+
+	def parse(urls=[], response)
+		lines = response.split("\n")
+
+		if (urls.length != lines.length)
+			error("Number of URLs in the reponse does not match the number of URLs in the request");
+			error("#{urls.length} / #{lines.length}");
+			error(response);
+			return errors(urls);
+		end
+
+		results = { }
+		for i in (0..lines.length - 1)
+			results[urls[i]] = lines[i]
+			debug(urls[i] + " => " + lines[i])
+		end
+
+		return results
+	end
+
+	
+	def errors(urls=[])
+		return Hash[*urls.map {|url| [url, 'error']}.flatten]
+	end
+
+	def ok(urls=[])
+		return Hash[*urls.map {|url| [url, 'ok']}.flatten]
+	end
+
+
+	def debug(message='')
+		puts message if (@debug == true)
+	end
+
+
+	def error(message='')
+		puts "#{message}\n" if (@debug == true or @error == true)
+		@last_error = message
+	end
+
+end

data/test/helper.rb

@@ -0,0 +1,32 @@
+require 'rubygems'
+require 'bundler'
+begin
+  Bundler.setup(:default, :development)
+rescue Bundler::BundlerError => e
+  $stderr.puts e.message
+  $stderr.puts "Run `bundle install` to install missing gems"
+  exit e.status_code
+end
+require 'test/unit'
+require 'shoulda'
+
+$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
+$LOAD_PATH.unshift(File.dirname(__FILE__))
+require 'google-safe-browsing-lookup'
+
+class Test::Unit::TestCase
+end
+
+class MockClient < SafeBrowsingLookup
+	def errors(urls=[])
+		super(urls)
+	end
+
+	def ok(urls=[])
+		super(urls)
+	end
+
+	def parse(urls=[], response)
+		super(urls, response)
+	end
+end

data/test/test_google-safe-browsing-lookup.rb

@@ -0,0 +1,72 @@
+require 'helper'
+
+class TestGoogleSafeBrowsingLookup < Test::Unit::TestCase
+  	context "Lookup client" do
+    	setup do
+      		@client = SafeBrowsingLookup.new('put_the_Google_API_key_here')
+	  		@mock = MockClient.new('put_the_Google_API_key_here')
+    	end
+
+
+		should "create a hash URL => 'error'" do
+			results = @mock.errors(['http://www.google.com/', 'http://www.google.org/'])
+
+			assert_equal 2, 		results.length, 					"Number of keys should match number of URLs"
+			assert_equal 'error', 	results['http://www.google.com/'], 	"An error should be returned"
+			assert_equal 'error', 	results['http://www.google.org/'], 	"An error should be returned"
+		end
+
+		should "create a hash URL => 'ok'" do
+			results = @mock.ok(['http://www.google.com/', 'http://www.google.org/'])
+
+			assert_equal 2, 	results.length, 						"Number of keys should match number of URLs"
+			assert_equal 'ok', 	results['http://www.google.com/'], 		"ok should be returned"
+			assert_equal 'ok', 	results['http://www.google.org/'], 		"ok should be returned"
+		end
+
+		should "parse the server response" do
+			response = "ok\nphishing"
+
+			results = @mock.parse(['http://www.google.com/', 'http://www.google.org/'], response)
+
+			assert_equal 2, 			results.length, 					"Number of keys should match number of URLs"
+			assert_equal 'ok', 			results['http://www.google.com/'], 	"ok should be returned"
+			assert_equal 'phishing', 	results['http://www.google.org/'], 	"phishing should be returned"
+		end
+
+		should "create find an error with server response" do
+			response = "ok\nphishing\nok"
+
+			results = @mock.parse(['http://www.google.com/', 'http://www.google.org/'], response)
+
+			assert_equal 2, 		results.length, 					"Number of keys should match number of URLs"
+			assert_equal 'error', 	results['http://www.google.com/'], 	"An error should be returned"
+			assert_equal 'error', 	results['http://www.google.org/'], 	"An error should be returned"
+		end
+
+		should "do live tests" do
+			if (@client.key == 'put_the_Google_API_key_here')
+				return # An valid key is required
+			end
+
+			results = @client.lookup('http://www.gumblar.cn/')
+			assert_equal 1, 		results.length, 					"Number of keys should match number of URLs"
+			assert_equal 'malware', results['http://www.gumblar.cn/'], 	"malware should be returned"
+		end
+
+		should "send more than 500 URLs" do
+			if (@client.key == 'put_the_Google_API_key_here')
+				return # An valid key is required
+			end
+
+			urls = []
+			(1..600).each do |id|
+				urls.push("http://www.gumblar.cn/#{id}")
+			end
+
+			results = @client.lookup(urls)
+			assert_equal urls.length, 	results.length, 					"Number of keys should match number of URLs"
+			assert_equal 'malware', 	results['http://www.gumblar.cn/1'], "malware should be returned"
+		end
+  	end
+end

metadata

@@ -0,0 +1,115 @@
+--- !ruby/object:Gem::Specification
+name: google-safe-browsing-lookup
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+  prerelease: 
+platform: ruby
+authors:
+- Julien Sobrier
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2011-12-14 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: url
+  requirement: &130589580 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: *130589580
+- !ruby/object:Gem::Dependency
+  name: shoulda
+  requirement: &130589080 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: *130589080
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: &130588580 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 1.0.0
+  type: :development
+  prerelease: false
+  version_requirements: *130588580
+- !ruby/object:Gem::Dependency
+  name: jeweler
+  requirement: &130588100 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 1.6.4
+  type: :development
+  prerelease: false
+  version_requirements: *130588100
+- !ruby/object:Gem::Dependency
+  name: rcov
+  requirement: &130587600 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: *130587600
+description: Google Safe Browsing v2 Lookup is a lighter alternative to the full API.
+  The service allows users to check up to 10,000 URLs a day against their list og
+  phishing and malicious sites. A free API key is required.
+email: julien@sobrier.net
+executables: []
+extensions: []
+extra_rdoc_files:
+- LICENSE.txt
+- README.rdoc
+files:
+- Gemfile
+- LICENSE.txt
+- README.rdoc
+- Rakefile
+- VERSION
+- lib/google-safe-browsing-lookup.rb
+- test/helper.rb
+- test/test_google-safe-browsing-lookup.rb
+homepage: http://github.com/juliensobrier/google-safe-browsing-lookup
+licenses:
+- MIT
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+      segments:
+      - 0
+      hash: 3258559681229541740
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.11
+signing_key: 
+specification_version: 3
+summary: Ruby library for the Google Safe Browsing v2 Lookup API
+test_files: []