linkingpaths-ruby-web-search 0.0.2
Sign up to get free protection for your applications and to get access to all the features.
- data/LICENSE +20 -0
- data/README.markdown +37 -0
- data/Rakefile +58 -0
- data/lib/curbemu.rb +68 -0
- data/lib/ruby-web-search.rb +194 -0
- data/spec/ruby-web-search-unthreaded.rb +88 -0
- data/spec/ruby-web-search_spec.rb +88 -0
- data/spec/spec_helper.rb +3 -0
- metadata +69 -0
data/LICENSE
ADDED
@@ -0,0 +1,20 @@
|
|
1
|
+
Copyright (c) 2009 Matt Aimonetti
|
2
|
+
|
3
|
+
Permission is hereby granted, free of charge, to any person obtaining
|
4
|
+
a copy of this software and associated documentation files (the
|
5
|
+
"Software"), to deal in the Software without restriction, including
|
6
|
+
without limitation the rights to use, copy, modify, merge, publish,
|
7
|
+
distribute, sublicense, and/or sell copies of the Software, and to
|
8
|
+
permit persons to whom the Software is furnished to do so, subject to
|
9
|
+
the following conditions:
|
10
|
+
|
11
|
+
The above copyright notice and this permission notice shall be
|
12
|
+
included in all copies or substantial portions of the Software.
|
13
|
+
|
14
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
15
|
+
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
16
|
+
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
17
|
+
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
|
18
|
+
LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
|
19
|
+
OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
|
20
|
+
WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
data/README.markdown
ADDED
@@ -0,0 +1,37 @@
|
|
1
|
+
# Ruby Web Search
|
2
|
+
|
3
|
+
This gem allows you to query google search engine from Ruby.
|
4
|
+
So far, only Google is supported.
|
5
|
+
|
6
|
+
|
7
|
+
Simple example on how to query Google:
|
8
|
+
|
9
|
+
>> require 'ruby-web-search'
|
10
|
+
=> true
|
11
|
+
>> response = RubyWebSearch::Google.search(:query => "Natalie Portman")
|
12
|
+
>> response.results
|
13
|
+
=> [{:content=>"<b>Natalie Portman</b>, Star Wars, Phantom Menace, Attack of the Clones, Amidala, Leon, Professional, Where The Heart Is, Anywhere But Here, Seagull, Heat, <b>...</b>", :title=>"Natalie Portman . Com - News", :url=>"http://www.natalieportman.com/", :domain=>"www.natalieportman.com", :cache_url=>"http://www.google.com/search?q=cache:9hGoJVGBJ2sJ:www.natalieportman.com"}, {:content=>"<b>Natalie Portman</b> was born on June 9th, 1981 in Jerusalem, Israel, as the... Visit IMDb for Photos, Filmography, Discussions, Bio, News, Awards, Agent, <b>...</b>", :title=>"Natalie Portman", :url=>"http://www.imdb.com/name/nm0000204/", :domain=>"www.imdb.com", :cache_url=>"http://www.google.com/search?q=cache:JLzGjsYYdlkJ:www.imdb.com"}, {:content=>"<b>Natalie Portman</b> (Hebrew: \327\240\327\230\327\234\327\231 \327\244\327\225\327\250\327\230\327\236\327\237\342\200\216; born <b>Natalie</b> Hershlag June 9, 1981) is an Israeli-American actress. <b>Portman</b> began her career in the early 1990s, <b>...</b>", :title=>"Natalie Portman - Wikipedia, the free encyclopedia", :url=>"http://en.wikipedia.org/wiki/Natalie_Portman", :domain=>"en.wikipedia.org", :cache_url=>"http://www.google.com/search?q=cache:32A4VEkC23gJ:en.wikipedia.org"}, {:content=>"Aug 30, 2008 <b>...</b> media on Miss <b>Portman</b>. You may recognize <b>Natalie</b> for her roles in <b>....</b> is in in no way affiliated with <b>Natalie Portman</b> or her management. <b>...</b>", :title=>"Natalie Portman ORG ++{natalie-p.org} | your premiere NATALIE ...", :url=>"http://www.natalie-p.org/", :domain=>"www.natalie-p.org", :cache_url=>"http://www.google.com/search?q=cache:wv-CVcMW2SEJ:www.natalie-p.org"}]
|
14
|
+
|
15
|
+
A google search returns a Response instance. Call `results` on the response to get the array on result.
|
16
|
+
A Result is a simple hash object with few keys available:
|
17
|
+
|
18
|
+
* title Title of the result
|
19
|
+
* url Url of the result
|
20
|
+
* domain Root url of the result
|
21
|
+
* content Snippet of the result content
|
22
|
+
* cache\_url Google cache url
|
23
|
+
|
24
|
+
|
25
|
+
By default, only the 4 top results get retrieved, you can specify the exact amount of results you want by passing the size argument.
|
26
|
+
RubyWebSearch::Google.search(:query => "Natalie Portman", :size => 10)
|
27
|
+
|
28
|
+
## TODO
|
29
|
+
|
30
|
+
* Full support of the google api
|
31
|
+
* support more search engines (Yahoo, live etc...)
|
32
|
+
|
33
|
+
## Experimentations
|
34
|
+
|
35
|
+
Here are some benchmarks, it looks like running multiple concurrent threads is often not worth it
|
36
|
+
http://gist.github.com/45350
|
37
|
+
warmed up jruby benchmarks
|
data/Rakefile
ADDED
@@ -0,0 +1,58 @@
|
|
1
|
+
require 'rubygems'
|
2
|
+
require 'rake/gempackagetask'
|
3
|
+
require 'rubygems/specification'
|
4
|
+
require 'date'
|
5
|
+
require 'spec/rake/spectask'
|
6
|
+
|
7
|
+
GEM = "ruby-web-search"
|
8
|
+
GEM_VERSION = "0.0.2"
|
9
|
+
AUTHOR = "Matt Aimonetti"
|
10
|
+
EMAIL = "mattaimonetti@gmail.com"
|
11
|
+
HOMEPAGE = "http://merbist.com"
|
12
|
+
SUMMARY = "A Ruby gem that provides a way to retrieve search results via the main search engines using Ruby"
|
13
|
+
|
14
|
+
spec = Gem::Specification.new do |s|
|
15
|
+
s.name = GEM
|
16
|
+
s.version = GEM_VERSION
|
17
|
+
s.platform = Gem::Platform::RUBY
|
18
|
+
s.has_rdoc = true
|
19
|
+
s.extra_rdoc_files = ["LICENSE"]
|
20
|
+
s.summary = SUMMARY
|
21
|
+
s.description = s.summary
|
22
|
+
s.author = AUTHOR
|
23
|
+
s.email = EMAIL
|
24
|
+
s.homepage = HOMEPAGE
|
25
|
+
|
26
|
+
# Uncomment this to add a dependency
|
27
|
+
# s.add_dependency "curb"
|
28
|
+
s.add_dependency "json"
|
29
|
+
|
30
|
+
s.require_path = 'lib'
|
31
|
+
s.autorequire = GEM
|
32
|
+
s.files = %w(LICENSE README.markdown Rakefile) + Dir.glob("{lib,spec}/**/*")
|
33
|
+
end
|
34
|
+
|
35
|
+
task :default => :spec
|
36
|
+
|
37
|
+
desc "Run specs"
|
38
|
+
Spec::Rake::SpecTask.new do |t|
|
39
|
+
t.spec_files = FileList['spec/**/*_spec.rb']
|
40
|
+
t.spec_opts = %w(-fs --color)
|
41
|
+
end
|
42
|
+
|
43
|
+
|
44
|
+
Rake::GemPackageTask.new(spec) do |pkg|
|
45
|
+
pkg.gem_spec = spec
|
46
|
+
end
|
47
|
+
|
48
|
+
desc "install the gem locally"
|
49
|
+
task :install => [:package] do
|
50
|
+
sh %{sudo gem install pkg/#{GEM}-#{GEM_VERSION}}
|
51
|
+
end
|
52
|
+
|
53
|
+
desc "create a gemspec file"
|
54
|
+
task :make_spec do
|
55
|
+
File.open("#{GEM}.gemspec", "w") do |file|
|
56
|
+
file.puts spec.to_ruby
|
57
|
+
end
|
58
|
+
end
|
data/lib/curbemu.rb
ADDED
@@ -0,0 +1,68 @@
|
|
1
|
+
require 'net/http'
|
2
|
+
|
3
|
+
module Curl
|
4
|
+
module Err
|
5
|
+
class CurlError < RuntimeError; end
|
6
|
+
class GotNothingError < CurlError; end
|
7
|
+
class ConnectionFailedError < CurlError; end
|
8
|
+
class TimeoutError < CurlError; end
|
9
|
+
class HttpError < CurlError; end
|
10
|
+
end
|
11
|
+
class Easy
|
12
|
+
attr_accessor :timeout, :url, :body_str, :headers, :conn
|
13
|
+
|
14
|
+
def initialize(url = nil)
|
15
|
+
@url = url
|
16
|
+
@headers = {}
|
17
|
+
@body_str = nil
|
18
|
+
end
|
19
|
+
|
20
|
+
#Not yet implemented.. only needed for importing from LibraryThing
|
21
|
+
def header_str
|
22
|
+
""
|
23
|
+
end
|
24
|
+
|
25
|
+
#Curl::Easy.perform("http://old-xisbn.oclc.org/xid/isbn/1234").body_str
|
26
|
+
#Curl::Easy.perform("http://old-xisbn.oclc.org/xid/isbn/1234").header_str
|
27
|
+
def self.perform(url)
|
28
|
+
c = self.new(url)
|
29
|
+
yield(c) if block_given?
|
30
|
+
c.perform
|
31
|
+
c
|
32
|
+
end
|
33
|
+
|
34
|
+
def self.http_get(url)
|
35
|
+
c = self.new(url)
|
36
|
+
yield(c) if block_given?
|
37
|
+
c.perform
|
38
|
+
c
|
39
|
+
end
|
40
|
+
|
41
|
+
#Curl::Easy.http_post("http://foo.com", {"img_url" => url}) { |r| r.headers = 'Content-Type: text/json' }.body_str)
|
42
|
+
def self.http_post(url, options = {})
|
43
|
+
c = self.new(url)
|
44
|
+
yield(c) if block_given?
|
45
|
+
c.http_post(options)
|
46
|
+
c
|
47
|
+
end
|
48
|
+
|
49
|
+
def perform
|
50
|
+
uri = URI.parse(url)
|
51
|
+
res = Net::HTTP.start(uri.host, uri.port) {|http|
|
52
|
+
http.request(Net::HTTP::Get.new(uri.request_uri))
|
53
|
+
}
|
54
|
+
@body_str = res.body
|
55
|
+
rescue => e
|
56
|
+
raise ::Curl::Err::HttpError, e.message
|
57
|
+
end
|
58
|
+
|
59
|
+
def http_post(options = {})
|
60
|
+
uri = URI.parse(url)
|
61
|
+
http = Net::HTTP.new(uri.host, uri.port)
|
62
|
+
resp, data = http.post(uri.request_uri, options, headers)
|
63
|
+
@body_str = data
|
64
|
+
rescue => e
|
65
|
+
raise ::Curl::Err::HttpError, e.message
|
66
|
+
end
|
67
|
+
end
|
68
|
+
end
|
@@ -0,0 +1,194 @@
|
|
1
|
+
require 'rubygems'
|
2
|
+
require 'cgi'
|
3
|
+
require 'json'
|
4
|
+
|
5
|
+
# begin
|
6
|
+
# gem 'curb'
|
7
|
+
# require 'curb'
|
8
|
+
# rescue LoadError
|
9
|
+
require File.join(File.dirname(__FILE__), 'curbemu')
|
10
|
+
# end
|
11
|
+
|
12
|
+
|
13
|
+
$RUBY_WEB_SEARCH_DEBUG = false
|
14
|
+
|
15
|
+
class RubyWebSearch
|
16
|
+
|
17
|
+
# http://code.google.com/apis/ajaxsearch/documentation/reference.html
|
18
|
+
class Google
|
19
|
+
|
20
|
+
def self.search(options={})
|
21
|
+
query = ::RubyWebSearch::Google::Query.new(options)
|
22
|
+
query.execute
|
23
|
+
end
|
24
|
+
|
25
|
+
def self.unthreaded_search(options={})
|
26
|
+
query = ::RubyWebSearch::Google::Query.new(options)
|
27
|
+
query.execute_unthreaded
|
28
|
+
end
|
29
|
+
|
30
|
+
class Query
|
31
|
+
attr_accessor :query, :start_index, :result_size, :filter, :country_code, :language_code
|
32
|
+
attr_accessor :safe_search, :type, :custom_search_engine_id, :version, :referer, :request_url
|
33
|
+
attr_accessor :size, :cursor, :custom_request_url, :response
|
34
|
+
|
35
|
+
class Error < StandardError; end
|
36
|
+
|
37
|
+
SEARCH_BASE_URLS = { :web => "http://ajax.googleapis.com/ajax/services/search/web",
|
38
|
+
:local => "http://ajax.googleapis.com/ajax/services/search/local",
|
39
|
+
:video => "http://ajax.googleapis.com/ajax/services/search/video",
|
40
|
+
:blog => "http://ajax.googleapis.com/ajax/services/search/blogs",
|
41
|
+
:news => "http://ajax.googleapis.com/ajax/services/search/news",
|
42
|
+
:book => "http://ajax.googleapis.com/ajax/services/search/books",
|
43
|
+
:image => "http://ajax.googleapis.com/ajax/services/search/images",
|
44
|
+
:patent => "http://ajax.googleapis.com/ajax/services/search/patent"
|
45
|
+
}
|
46
|
+
|
47
|
+
#
|
48
|
+
# You can overwrite the query building process by passing the request url to use.
|
49
|
+
#
|
50
|
+
# ==== Params
|
51
|
+
# query<String>
|
52
|
+
# start_index<Integer>
|
53
|
+
# result_size<String> small or large (4 or 8 results) default: small
|
54
|
+
# filter
|
55
|
+
# country_code<String> 2 letters language code for the country you want
|
56
|
+
# to limit to
|
57
|
+
# language_code<String> (Web only)
|
58
|
+
# safe_search<String> active, moderate or off. Default: active (web only)
|
59
|
+
# custom_search_engine_id<String> optional argument supplying the unique id for
|
60
|
+
# the Custom Search Engine that should be used for the request (e.g., 000455696194071821846:reviews).
|
61
|
+
# (web only)
|
62
|
+
#
|
63
|
+
def initialize(options={})
|
64
|
+
if options[:custom_request_url]
|
65
|
+
@custom_request_url = options[:request_url]
|
66
|
+
else
|
67
|
+
@query = options[:query]
|
68
|
+
raise Google::Query::Error, "You need to pass a query" unless @query
|
69
|
+
@cursor = options[:start_index] || 0
|
70
|
+
@result_size = options[:result_size]
|
71
|
+
@filter = options[:filter]
|
72
|
+
@type = options[:type] || :web
|
73
|
+
@country_code = options[:country_code] ? "country#{options[:country_code].upcase}" : nil
|
74
|
+
@language_code = options[:language_code] ? "lang_#{options[:language_code].upcase}" : nil
|
75
|
+
@safe_search = options[:safe_search]
|
76
|
+
@custom_search_engine_id = options[:custom_search_engine_id]
|
77
|
+
@version = options[:version] || "1.0"
|
78
|
+
@referer = options[:referer] || "http://github.com/mattetti/"
|
79
|
+
@size = options[:size] || 4
|
80
|
+
@result_size = "large" if size > 4 # increase the result set size to avoid making too many requests
|
81
|
+
@size = 8 if (@result_size == "large" && size < 8)
|
82
|
+
end
|
83
|
+
@response ||= Response.new(:query => (query || custom_request_url), :size => size)
|
84
|
+
end
|
85
|
+
|
86
|
+
def build_request
|
87
|
+
if custom_request_url
|
88
|
+
custom_request_url
|
89
|
+
else
|
90
|
+
@request_url = "#{SEARCH_BASE_URLS[type]}?v=#{version}&q=#{CGI.escape(query)}"
|
91
|
+
@request_url << "&rsz=#{result_size}" if result_size
|
92
|
+
@request_url << "&start=#{cursor}" if cursor > 0
|
93
|
+
@request_url << "&hl=#{language_code}" if language_code
|
94
|
+
|
95
|
+
puts request_url if $RUBY_WEB_SEARCH_DEBUG
|
96
|
+
request_url
|
97
|
+
end
|
98
|
+
end
|
99
|
+
|
100
|
+
def build_requests
|
101
|
+
if custom_request_url
|
102
|
+
requests = [custom_request_url]
|
103
|
+
else
|
104
|
+
requests = []
|
105
|
+
# create an array of requests based on the fact that google limits
|
106
|
+
# us to 8 responses per request but let us use a cursor
|
107
|
+
(size / 8.to_f).ceil.times do |n|
|
108
|
+
url = "#{SEARCH_BASE_URLS[type]}?v=#{version}&q=#{CGI.escape(query)}"
|
109
|
+
url << "&rsz=#{result_size}" if result_size
|
110
|
+
url << "&hl=#{language_code}" if language_code
|
111
|
+
url << "&start=#{cursor}"
|
112
|
+
@cursor += 8
|
113
|
+
requests << url
|
114
|
+
end
|
115
|
+
|
116
|
+
puts requests.inspect if $RUBY_WEB_SEARCH_DEBUG
|
117
|
+
requests
|
118
|
+
end
|
119
|
+
end
|
120
|
+
|
121
|
+
# Makes the request to Google
|
122
|
+
# if a larger set was requested than what is returned,
|
123
|
+
# more requests are made until the correct amount is available
|
124
|
+
def execute_unthreaded
|
125
|
+
@curl_request ||= ::Curl::Easy.new(){ |curl| curl.headers["Referer"] = referer }
|
126
|
+
@curl_request.url = build_request
|
127
|
+
@curl_request.perform
|
128
|
+
results = JSON.load(@curl_request.body_str)
|
129
|
+
|
130
|
+
response.process(results)
|
131
|
+
@cursor = response.results.size - 1
|
132
|
+
if ((cursor + 1) < size && custom_request_url.nil?)
|
133
|
+
puts "cursor: #{cursor} requested results size: #{size}" if $RUBY_WEB_SEARCH_DEBUG
|
134
|
+
execute_unthreaded
|
135
|
+
else
|
136
|
+
response.limit(size)
|
137
|
+
end
|
138
|
+
end
|
139
|
+
|
140
|
+
# Makes the request to Google
|
141
|
+
# if a larger set was requested than what is returned,
|
142
|
+
# more requests are made until the correct amount is available
|
143
|
+
def execute
|
144
|
+
threads = build_requests.map do |req|
|
145
|
+
Thread.new do
|
146
|
+
curl_request = ::Curl::Easy.new(req){ |curl| curl.headers["Referer"] = referer }
|
147
|
+
curl_request.perform
|
148
|
+
JSON.load(curl_request.body_str)
|
149
|
+
end
|
150
|
+
end
|
151
|
+
threads.each do |t|
|
152
|
+
response.process(t.value)
|
153
|
+
end
|
154
|
+
response.limit(size)
|
155
|
+
end
|
156
|
+
|
157
|
+
end #of Query
|
158
|
+
|
159
|
+
|
160
|
+
class Response
|
161
|
+
attr_reader :results, :status, :query, :size, :estimated_result_count
|
162
|
+
def initialize(google_raw_response={})
|
163
|
+
process(google_raw_response) unless google_raw_response.empty?
|
164
|
+
end
|
165
|
+
|
166
|
+
def process(google_raw_response={})
|
167
|
+
@query ||= google_raw_response[:query]
|
168
|
+
@size ||= google_raw_response[:size]
|
169
|
+
@results ||= []
|
170
|
+
@status = google_raw_response["responseStatus"]
|
171
|
+
if google_raw_response["responseData"] && status && status == 200
|
172
|
+
estimated_result_count ||= google_raw_response["cursor"]["estimatedResultCount"] if google_raw_response["cursor"]
|
173
|
+
@results += google_raw_response["responseData"]["results"].map do |r|
|
174
|
+
{
|
175
|
+
:title => r["titleNoFormatting"],
|
176
|
+
:url => r["unescapedUrl"],
|
177
|
+
:cache_url => r["cacheUrl"],
|
178
|
+
:content => r["content"],
|
179
|
+
:domain => r["visibleUrl"]
|
180
|
+
}
|
181
|
+
end
|
182
|
+
end
|
183
|
+
|
184
|
+
def limit(req_size)
|
185
|
+
@results = @results[0...req_size]
|
186
|
+
self
|
187
|
+
end
|
188
|
+
|
189
|
+
end
|
190
|
+
end #of Response
|
191
|
+
|
192
|
+
end #of Google
|
193
|
+
|
194
|
+
end
|
@@ -0,0 +1,88 @@
|
|
1
|
+
require File.dirname(__FILE__) + '/spec_helper'
|
2
|
+
$RUBY_WEB_SEARCH_DEBUG = true
|
3
|
+
|
4
|
+
describe "ruby-web-search" do
|
5
|
+
|
6
|
+
describe "Google search" do
|
7
|
+
|
8
|
+
describe "simple format" do
|
9
|
+
before(:all) do
|
10
|
+
@response = RubyWebSearch::Google.unthreaded_search(:query => "Natalie Portman")
|
11
|
+
end
|
12
|
+
|
13
|
+
it "should return a RubyWebSeach::Google::Response " do
|
14
|
+
@response.should be_an_instance_of(RubyWebSearch::Google::Response)
|
15
|
+
end
|
16
|
+
|
17
|
+
it "should have results" do
|
18
|
+
@response.results.should be_an_instance_of(Array)
|
19
|
+
@response.results.first.should be_an_instance_of(Hash)
|
20
|
+
end
|
21
|
+
|
22
|
+
it "should have 4 results (small request set size)" do
|
23
|
+
@response.results.size.should == 4
|
24
|
+
end
|
25
|
+
|
26
|
+
describe "results" do
|
27
|
+
before(:all) do
|
28
|
+
@results = @response.results
|
29
|
+
end
|
30
|
+
|
31
|
+
it "should have a title" do
|
32
|
+
@results.first[:title].should be_an_instance_of(String)
|
33
|
+
@results.first[:title].size.should > 3
|
34
|
+
end
|
35
|
+
|
36
|
+
it "should have an url" do
|
37
|
+
@results.first[:url].should be_an_instance_of(String)
|
38
|
+
@results.first[:url].size.should > 3
|
39
|
+
end
|
40
|
+
|
41
|
+
it "should have a cache url" do
|
42
|
+
@results.first[:cache_url].should be_an_instance_of(String)
|
43
|
+
@results.first[:cache_url].size.should > 3
|
44
|
+
end
|
45
|
+
|
46
|
+
it "should have content" do
|
47
|
+
@results.first[:content].should be_an_instance_of(String)
|
48
|
+
@results.first[:content].size.should > 15
|
49
|
+
end
|
50
|
+
|
51
|
+
it "should have a domain" do
|
52
|
+
@results.first[:domain].should be_an_instance_of(String)
|
53
|
+
@results.first[:domain].size.should > 7
|
54
|
+
@results.first[:url].should include(@response.results.first[:domain])
|
55
|
+
end
|
56
|
+
end
|
57
|
+
end
|
58
|
+
|
59
|
+
describe "large result set" do
|
60
|
+
before(:all) do
|
61
|
+
@response = RubyWebSearch::Google.unthreaded_search(:query => "Natalie Portman", :result_size => "large")
|
62
|
+
end
|
63
|
+
|
64
|
+
it "should have 8 results" do
|
65
|
+
@response.results.size.should == 8
|
66
|
+
end
|
67
|
+
end
|
68
|
+
|
69
|
+
describe "custom size result set" do
|
70
|
+
before(:all) do
|
71
|
+
@response = RubyWebSearch::Google.unthreaded_search(:query => "Natalie Portman", :size => 24)
|
72
|
+
@results = @response.results
|
73
|
+
end
|
74
|
+
|
75
|
+
it "should have exactly 24 results" do
|
76
|
+
@results.size.should == 24
|
77
|
+
end
|
78
|
+
|
79
|
+
it "should have 24 unique results" do
|
80
|
+
first = @results.shift
|
81
|
+
@results.each do |result|
|
82
|
+
first[:url].should_not == result[:url]
|
83
|
+
end
|
84
|
+
end
|
85
|
+
end
|
86
|
+
end
|
87
|
+
|
88
|
+
end
|
@@ -0,0 +1,88 @@
|
|
1
|
+
require File.dirname(__FILE__) + '/spec_helper'
|
2
|
+
$RUBY_WEB_SEARCH_DEBUG = true
|
3
|
+
|
4
|
+
describe "ruby-web-search" do
|
5
|
+
|
6
|
+
describe "Google search" do
|
7
|
+
|
8
|
+
describe "simple format" do
|
9
|
+
before(:all) do
|
10
|
+
@response = RubyWebSearch::Google.search(:query => "Natalie Portman")
|
11
|
+
end
|
12
|
+
|
13
|
+
it "should return a RubyWebSeach::Google::Response " do
|
14
|
+
@response.should be_an_instance_of(RubyWebSearch::Google::Response)
|
15
|
+
end
|
16
|
+
|
17
|
+
it "should have results" do
|
18
|
+
@response.results.should be_an_instance_of(Array)
|
19
|
+
@response.results.first.should be_an_instance_of(Hash)
|
20
|
+
end
|
21
|
+
|
22
|
+
it "should have 4 results (small request set size)" do
|
23
|
+
@response.results.size.should == 4
|
24
|
+
end
|
25
|
+
|
26
|
+
describe "results" do
|
27
|
+
before(:all) do
|
28
|
+
@results = @response.results
|
29
|
+
end
|
30
|
+
|
31
|
+
it "should have a title" do
|
32
|
+
@results.first[:title].should be_an_instance_of(String)
|
33
|
+
@results.first[:title].size.should > 3
|
34
|
+
end
|
35
|
+
|
36
|
+
it "should have an url" do
|
37
|
+
@results.first[:url].should be_an_instance_of(String)
|
38
|
+
@results.first[:url].size.should > 3
|
39
|
+
end
|
40
|
+
|
41
|
+
it "should have a cache url" do
|
42
|
+
@results.first[:cache_url].should be_an_instance_of(String)
|
43
|
+
@results.first[:cache_url].size.should > 3
|
44
|
+
end
|
45
|
+
|
46
|
+
it "should have content" do
|
47
|
+
@results.first[:content].should be_an_instance_of(String)
|
48
|
+
@results.first[:content].size.should > 15
|
49
|
+
end
|
50
|
+
|
51
|
+
it "should have a domain" do
|
52
|
+
@results.first[:domain].should be_an_instance_of(String)
|
53
|
+
@results.first[:domain].size.should > 7
|
54
|
+
@results.first[:url].should include(@response.results.first[:domain])
|
55
|
+
end
|
56
|
+
end
|
57
|
+
end
|
58
|
+
|
59
|
+
describe "large result set" do
|
60
|
+
before(:all) do
|
61
|
+
@response = RubyWebSearch::Google.search(:query => "Natalie Portman", :result_size => "large")
|
62
|
+
end
|
63
|
+
|
64
|
+
it "should have 8 results" do
|
65
|
+
@response.results.size.should == 8
|
66
|
+
end
|
67
|
+
end
|
68
|
+
|
69
|
+
describe "custom size result set" do
|
70
|
+
before(:all) do
|
71
|
+
@response = RubyWebSearch::Google.search(:query => "Natalie Portman", :size => 24)
|
72
|
+
@results = @response.results
|
73
|
+
end
|
74
|
+
|
75
|
+
it "should have exactly 24 results" do
|
76
|
+
@results.size.should == 24
|
77
|
+
end
|
78
|
+
|
79
|
+
it "should have 24 unique results" do
|
80
|
+
first = @results.shift
|
81
|
+
@results.each do |result|
|
82
|
+
first[:url].should_not == result[:url]
|
83
|
+
end
|
84
|
+
end
|
85
|
+
end
|
86
|
+
end
|
87
|
+
|
88
|
+
end
|
data/spec/spec_helper.rb
ADDED
metadata
ADDED
@@ -0,0 +1,69 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: linkingpaths-ruby-web-search
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.0.2
|
5
|
+
platform: ruby
|
6
|
+
authors:
|
7
|
+
- Matt Aimonetti
|
8
|
+
autorequire: ruby-web-search
|
9
|
+
bindir: bin
|
10
|
+
cert_chain: []
|
11
|
+
|
12
|
+
date: 2009-05-16 00:00:00 -07:00
|
13
|
+
default_executable:
|
14
|
+
dependencies:
|
15
|
+
- !ruby/object:Gem::Dependency
|
16
|
+
name: json
|
17
|
+
type: :runtime
|
18
|
+
version_requirement:
|
19
|
+
version_requirements: !ruby/object:Gem::Requirement
|
20
|
+
requirements:
|
21
|
+
- - ">="
|
22
|
+
- !ruby/object:Gem::Version
|
23
|
+
version: "0"
|
24
|
+
version:
|
25
|
+
description: A Ruby gem that provides a way to retrieve search results via the main search engines using Ruby
|
26
|
+
email: mattaimonetti@gmail.com
|
27
|
+
executables: []
|
28
|
+
|
29
|
+
extensions: []
|
30
|
+
|
31
|
+
extra_rdoc_files:
|
32
|
+
- LICENSE
|
33
|
+
files:
|
34
|
+
- LICENSE
|
35
|
+
- README.markdown
|
36
|
+
- Rakefile
|
37
|
+
- lib/curbemu.rb
|
38
|
+
- lib/ruby-web-search.rb
|
39
|
+
- spec/ruby-web-search-unthreaded.rb
|
40
|
+
- spec/ruby-web-search_spec.rb
|
41
|
+
- spec/spec_helper.rb
|
42
|
+
has_rdoc: true
|
43
|
+
homepage: http://merbist.com
|
44
|
+
post_install_message:
|
45
|
+
rdoc_options: []
|
46
|
+
|
47
|
+
require_paths:
|
48
|
+
- lib
|
49
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
50
|
+
requirements:
|
51
|
+
- - ">="
|
52
|
+
- !ruby/object:Gem::Version
|
53
|
+
version: "0"
|
54
|
+
version:
|
55
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
56
|
+
requirements:
|
57
|
+
- - ">="
|
58
|
+
- !ruby/object:Gem::Version
|
59
|
+
version: "0"
|
60
|
+
version:
|
61
|
+
requirements: []
|
62
|
+
|
63
|
+
rubyforge_project:
|
64
|
+
rubygems_version: 1.2.0
|
65
|
+
signing_key:
|
66
|
+
specification_version: 2
|
67
|
+
summary: A Ruby gem that provides a way to retrieve search results via the main search engines using Ruby
|
68
|
+
test_files: []
|
69
|
+
|