gscraper 0.1.7 → 0.2.0

data/lib/gscraper/sponsored_ad.rb

@@ -1,3 +1,25 @@
+#
+#--
+# GScraper - A web-scraping interface to various Google Services.
+#
+# Copyright (c) 2007-2008 Hal Brodigan (postmodern.mod3 at gmail.com)
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+#++
+#
+
 require 'gscraper/extensions/uri'
 
 module GScraper
@@ -14,14 +36,16 @@ module GScraper
     #
     def initialize(title,url)
       @title = title
-      @url = URI.parse(url)
+      @url = url
     end
 
     #
     # Returns the direct URL of the ad.
     #
     def direct_url
-      @url.query_params['adurl'] || @url.query_params['q']
+      uri = URI(@url)
+
+      return (uri.query_params['adurl'] || uri.query_params['q'])
     end
 
     #

data/lib/gscraper/sponsored_links.rb

@@ -1,12 +1,38 @@
+#
+#--
+# GScraper - A web-scraping interface to various Google Services.
+#
+# Copyright (c) 2007-2008 Hal Brodigan (postmodern.mod3 at gmail.com)
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+#++
+#
+
 require 'gscraper/sponsored_ad'
 
 module GScraper
   class SponsoredLinks < Array
     #
-    # Creates a new SponsoredLinks object with the given _ads_.
+    # Creates a new SponsoredLinks object with the given _ads_. If a
+    # _block_ is given, it will be passed the newly created SponsoredLinks
+    # object.
     #
-    def initialize(ads=[])
+    def initialize(ads=[],&block)
       super(ads)
+
+      block.call(self) if block
     end
 
     #
@@ -74,10 +100,6 @@ module GScraper
     #
     #   sponsored.ads_with_url(/\.com/) # => SponsoredLinks
     #
-    #   sponsored.ads_with_url(/^https:\/\//) do |ad|
-    #     puts ad.title
-    #   end
-    #
     def ads_with_url(url,&block)
       if url.kind_of?(Regexp)
         ads = ads_with { |ad| ad.url =~ url }
@@ -89,6 +111,24 @@ module GScraper
       return ads
     end
 
+    #
+    # Selects the ads with the matching _direct_url_. The _direct_url_ may
+    # be either a String or a Regexp. If _block_ is given, each matching
+    # ad will be passed to the _block_.
+    #
+    #   sponsored.ads_with_direct_url(/\.com/) # => SponsoredLinks
+    #
+    def ads_with_direct_url(direct_url,&block)
+      if direct_url.kind_of?(Regexp)
+        ads = ads_with { |ad| ad.direct_url =~ direct_url }
+      else
+        ads = ads_with { |ad| ad.direct_url == direct_url }
+      end
+
+      ads.each(&block) if block
+      return ads
+    end
+
     #
     # Returns an Array containing the titles of the ads within the
     # SponsoredLinks.
@@ -109,6 +149,16 @@ module GScraper
       map { |ad| ad.url }
     end
 
+    #
+    # Returns an Array containing the direct URLs of the ads within the
+    # SponsoredLinks.
+    #
+    #   sponsored.direct_urls # => [...]
+    #
+    def direct_urls
+      map { |ad| ad.direct_url }
+    end
+
     #
     # Iterates over each ad's title within the SponsoredLinks, passing each to
     # the given _block_.
@@ -120,7 +170,7 @@ module GScraper
     end
 
     #
-    # Iterates over each ad's url within the SponsoredLinks, passing each to
+    # Iterates over each ad's URL within the SponsoredLinks, passing each to
     # the given _block_.
     #
     #   each_url { |url| puts url }
@@ -129,6 +179,16 @@ module GScraper
       urls.each(&block)
     end
 
+    #
+    # Iterates over each ad's direct URL within the SponsoredLinks, passing
+    # each to the given _block_.
+    #
+    #   each_direct_url { |url| puts url }
+    #
+    def each_direct_url(&block)
+      direct_urls.each(&block)
+    end
+
     #
     # Returns the titles of the ads that match the specified _block_.
     #
@@ -139,7 +199,7 @@ module GScraper
     end
 
     #
-    # Returns the urls of the ads that match the specified _block_.
+    # Returns the URLs of the ads that match the specified _block_.
     #
     #   sponsored.urls_of { |ad| ad.title =~ /buy these pants/ }
     #
@@ -147,5 +207,14 @@ module GScraper
       ads_with(&block).urls
     end
 
+    #
+    # Returns the direct URLs of the ads that match the specified _block_.
+    #
+    #   sponsored.urls_of { |ad| ad.title =~ /buy these pants/ }
+    #
+    def direct_urls_of(&block)
+      ads_with(&block).direct_urls
+    end
+
   end
 end

data/lib/gscraper/version.rb

@@ -1,3 +1,25 @@
+#
+#--
+# GScraper - A web-scraping interface to various Google Services.
+#
+# Copyright (c) 2007-2008 Hal Brodigan (postmodern.mod3 at gmail.com)
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+#++
+#
+
 module GScraper
-  VERSION = '0.1.7'
+  VERSION = '0.2.0'
 end

data/spec/extensions/uri/http_spec.rb

@@ -0,0 +1,9 @@
+require 'spec_helper'
+
+require 'gscraper/extensions/uri'
+
+describe URI::HTTP do
+  it "should include QueryParams" do
+    URI::HTTP.include?(URI::QueryParams).should == true
+  end
+end

data/spec/extensions/uri/query_params_spec.rb

@@ -0,0 +1,38 @@
+require 'spec_helper'
+
+require 'gscraper/extensions/uri'
+
+describe "URI::QueryParams" do
+  before(:each) do
+    @uri = URI('http://www.test.com/page.php?x=1&y=one%20two&z')
+  end
+
+  it "should provide #query_params" do
+    @uri.should respond_to(:query_params)
+  end
+
+  it "#query_params should be a Hash" do
+    @uri.query_params.class.should == Hash
+  end
+
+  it "#query_params should contain params" do
+    @uri.query_params.empty?.should == false
+  end
+
+  it "#query_params can contain single-word params" do
+    @uri.query_params['x'].should == '1'
+  end
+
+  it "#query_params can contain multi-word params" do
+    @uri.query_params['y'].should == 'one two'
+  end
+
+  it "#query_params can contain empty params" do
+    @uri.query_params['z'].should be_nil
+  end
+
+  it "should update #query_params along with #query=" do
+    @uri.query = 'u=3'
+    @uri.query_params['u'].should == '3'
+  end
+end

data/spec/gscraper_spec.rb

@@ -0,0 +1,29 @@
+require 'spec_helper'
+
+require 'gscraper/gscraper'
+
+describe "GScraper" do
+  describe "User-Agent support" do
+    it "should have a default User-Agent string" do
+      GScraper.user_agent.should_not be_nil
+    end
+  end
+
+  describe "Proxy support" do
+    it "should provide a :host key" do
+      GScraper.proxy.has_key?(:host).should == true
+    end
+
+    it "should provide a :port key" do
+      GScraper.proxy.has_key?(:port).should == true
+    end
+
+    it "should provide a :user key" do
+      GScraper.proxy.has_key?(:user).should == true
+    end
+
+    it "should provide a :password key" do
+      GScraper.proxy.has_key?(:password).should == true
+    end
+  end
+end

data/spec/has_pages_examples.rb

@@ -0,0 +1,19 @@
+require 'spec_helper'
+
+shared_examples_for "has Pages" do
+
+  it "should have a first page" do
+    @query.first_page.should_not be_nil
+  end
+
+  it "should allow indexed access" do
+    @query[1].should_not be_nil
+  end
+
+  it "should allow accessing multiple pages" do
+    pages = @query.pages(1..2)
+    pages.should_not be_nil
+    pages.length.should == 2
+  end
+
+end

data/spec/has_sponsored_links_examples.rb

@@ -0,0 +1,57 @@
+require 'spec_helper'
+
+shared_examples_for "has Sponsored Links" do
+
+  it "should have ads" do
+    @links.length.should_not == 0
+  end
+
+  it "should have titles" do
+    @links.each_title do |title|
+      title.should_not be_nil
+    end
+  end
+
+  it "should have non-empty titles" do
+    @links.each_title do |title|
+      title.length.should_not == 0
+    end
+  end
+
+  it "should have URLs" do
+    @links.each_url do |url|
+      url.should_not be_nil
+    end
+  end
+
+  it "should have non-empty URLs" do
+    @links.each_url do |url|
+      url.length.should_not == 0
+    end
+  end
+
+  it "should have valid URLs" do
+    @links.each_url do |url|
+      url_should_be_valid(url)
+    end
+  end
+
+  it "should have direct URLs" do
+    @links.each_direct_url do |url|
+      url.should_not be_nil
+    end
+  end
+
+  it "should have non-empty direct URLs" do
+    @links.each_direct_url do |url|
+      url.length.should_not == 0
+    end
+  end
+
+  it "should have valid direct URLs" do
+    @links.each_direct_url do |url|
+      url_should_be_valid(url)
+    end
+  end
+
+end

data/spec/helpers/query.rb

@@ -0,0 +1 @@
+DEFAULT_QUERY = 'Ruby'

data/spec/helpers/uri.rb

@@ -0,0 +1,8 @@
+require 'uri'
+
+def url_should_be_valid(url)
+  uri = URI(url)
+  uri.scheme.should_not be_nil
+  uri.host.should_not be_nil
+  uri.path.should_not be_nil
+end

data/spec/page_has_results_examples.rb

@@ -0,0 +1,13 @@
+require 'spec_helper'
+
+shared_examples_for "Page has Results" do
+
+  it "should have results" do
+    @page.length.should_not == 0
+  end
+
+  it "should have the maximum amount of results per page" do
+    @page.length.should == @query.results_per_page
+  end
+
+end

data/spec/search/ajax_query_spec.rb

@@ -0,0 +1,124 @@
+require 'spec_helper'
+require 'has_pages_examples'
+require 'page_has_results_examples'
+require 'search/page_has_results_examples'
+
+require 'gscraper/search/ajax_query'
+
+describe GScraper::Search::AJAXQuery do
+
+  before(:all) do
+    @query = GScraper::Search::AJAXQuery.new(:query => DEFAULT_QUERY)
+    @page = @query.first_page
+  end
+
+  it_should_behave_like "has Pages"
+  it_should_behave_like "Page has Results"
+  it_should_behave_like "Page has Search Results"
+
+  describe "Search URL" do
+
+    before(:all) do
+      @uri = @query.search_url
+    end
+
+    it "should be a valid HTTP URI" do
+      @uri.class.should == URI::HTTP
+    end
+
+    it "should be a RESTful AJAX Search URL" do
+      @uri.path.should == '/uds/GwebSearch'
+    end
+
+    it "should have the default 'callback' query-param" do
+      callback = @uri.query_params['callback']
+      callback.should == 'google.search.WebSearch.RawCompletion'
+    end
+
+    it "should have the default 'context' query-param" do
+      @uri.query_params['context'].should == '0'
+    end
+
+    it "should have a default 'lstkp' query-param" do
+      @uri.query_params['lstkp'].should == '0'
+    end
+
+    it "should have a default 'rsz' query-param of 'large'" do
+      @uri.query_params['rsz'].should == 'large'
+    end
+
+    it "should have a default 'hl' query-param" do
+      hl = @uri.query_params['hl']
+      hl.should == GScraper::Search::AJAXQuery::DEFAULT_LANGUAGE
+    end
+
+    it "should have a default 'gss' query-param of '.com'" do
+      @uri.query_params['gss'].should == '.com'
+    end
+
+    it "should have a 'q' query-param" do
+      @uri.query_params['q'].should == DEFAULT_QUERY
+    end
+
+    it "should have a default 'sig' query-param" do
+      sig = @uri.query_params['sig']
+      sig.should == GScraper::Search::AJAXQuery::DEFAULT_SIG
+    end
+
+    it "should have a default 'key' query-param" do
+      key = @uri.query_params['key']
+      key.should == GScraper::Search::AJAXQuery::DEFAULT_KEY
+    end
+
+    it "should have a default 'v' query-param" do
+      v = @uri.query_params['v']
+      v.should == GScraper::Search::AJAXQuery::DEFAULT_VERSION
+    end
+
+  end
+
+  describe "page specific URLs" do
+
+    before(:all) do
+      @uri = @query.page_url(2)
+    end
+
+    it "should have a 'start' query-param" do
+      @uri.query_params['start'].should == @query.results_per_page
+    end
+
+  end
+
+  describe "queries from AJAX search URLs" do
+
+    before(:all) do
+      @version = '1.0'
+      @language = 'en'
+      @sig = '582c1116317355adf613a6a843f19ece'
+      @key = 'notsupplied'
+      @query = GScraper::Search::AJAXQuery.from_url("http://www.google.com/uds/GwebSearch?v=#{@version}&lstkp=0&rsz=large&hl=#{@language}&callback=google.search.WebSearch.RawCompletion&sig=#{@sig}&q=#{DEFAULT_QUERY}&gss=.com&context=0&key=#{@key}")
+    end
+
+    it "should have a version" do
+      @query.version.should == @version
+    end
+
+    it "should have a language" do
+      @query.language.should == @language
+    end
+
+    it "should have a sig" do
+      @query.sig.should == @sig
+    end
+
+    it "should have a key" do
+      @query.key.should == @key
+    end
+
+    it "should have a query" do
+      @query.query.should == DEFAULT_QUERY
+    end
+
+  end
+
+end