gscraper 0.1.0

data/History.txt

@@ -0,0 +1,5 @@
+== 0.1.0 / 2007-12-20
+
+ * Initial release.
+ * Supports the Google Search service.
+

data/LICENSE.txt

@@ -0,0 +1,23 @@
+
+
+The MIT License
+
+Copyright (c) 2007 Hal Brodigan
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/Manifest.txt

@@ -0,0 +1,17 @@
+History.txt
+LICENSE.txt
+Manifest.txt
+README.txt
+Rakefile
+lib/gscraper.rb
+lib/gscraper/gscraper.rb
+lib/gscraper/extensions/uri/http.rb
+lib/gscraper/extensions/uri.rb
+lib/gscraper/extensions.rb
+lib/gscraper/licenses.rb
+lib/gscraper/search/result.rb
+lib/gscraper/search/query.rb
+lib/gscraper/search/search.rb
+lib/gscraper/search.rb
+test/test_gscraper.rb
+test/search/query_from_url.rb

data/README.txt

@@ -0,0 +1,46 @@
+GScraper
+    by Postmodern Modulus III
+    http://rubyforge.net/projects/gscraper/
+
+== DESCRIPTION:
+  
+GScraper is a web-scraping interface to various Google Services.
+
+== FEATURES/PROBLEMS:
+  
+ * Supports the Google Search service.
+ * Provides HTTP access with custom User-Agent strings.
+
+== REQUIREMENTS:
+
+* Hpricot
+* Mechanize
+
+== INSTALL:
+
+sudo gem install gscraper
+
+== LICENSE:
+
+The MIT License
+
+Copyright (c) 2007 Hal Brodigan
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+'Software'), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/Rakefile

@@ -0,0 +1,17 @@
+# -*- ruby -*-
+
+require 'rubygems'
+require 'hoe'
+require './lib/gscraper/version.rb'
+
+Hoe.new('gscraper', GScraper::VERSION) do |p|
+  p.rubyforge_name = 'gscraper'
+  p.author = 'Postmodern Modulus III'
+  p.email = 'postmodern.mod3@gmail.com'
+  p.summary = 'A ruby web-scraping interface to various Google Services'
+  p.description = p.paragraphs_of('README.txt', 2..5).join("\n\n")
+  p.url = p.paragraphs_of('README.txt', 0).first.split(/\n/)[1..-1]
+  p.changes = p.paragraphs_of('History.txt', 0..1).join("\n\n")
+end
+
+# vim: syntax=Ruby

data/lib/gscraper.rb

@@ -0,0 +1,2 @@
+require 'gscraper/search'
+require 'gscraper/version'

data/lib/gscraper/extensions.rb

@@ -0,0 +1 @@
+require 'gscraper/extensions/uri'

data/lib/gscraper/extensions/uri.rb

@@ -0,0 +1 @@
+require 'gscraper/extensions/uri/http'

data/lib/gscraper/extensions/uri/http.rb

@@ -0,0 +1,71 @@
+module URI
+  class HTTP
+
+    # Query parameters
+    attr_reader :query_params
+
+    #
+    # Creates a new URI::HTTP object and initializes query_params as a
+    # new Hash.
+    #
+    def initialize(*args)
+      super(*args)
+
+      @query_params = {}
+      parse_query_params
+    end
+
+    #
+    # Sets the query data and updates query_params.
+    #
+    def query=(query_str)
+      new_query = super(query_str)
+      parse_query_params
+      return new_query
+    end
+
+    protected
+
+    #
+    # Parses the query parameters from the query data, populating
+    # query_params with the parsed parameters.
+    #
+    def parse_query_params
+      @query_params.clear
+
+      if @query
+        @query.split('&').each do |param|
+          name, value = param.split('=')
+
+          if value
+            @query_params[name] = URI.decode(value)
+          else
+            @query_params[name] = nil
+          end
+        end
+      end
+    end
+
+    private
+
+    # :nodoc
+    def path_query
+      str = @path
+
+      unless @query_params.empty?
+        str += '?' + @query_params.to_a.map { |name,value|
+          if value==true
+            "#{name}=active"
+          elsif value
+            "#{name}=#{URI.encode(value.to_s)}"
+          else
+            "#{name}="
+          end
+        }.join('&')
+      end
+
+      return str
+    end
+
+  end
+end

data/lib/gscraper/gscraper.rb

@@ -0,0 +1,62 @@
+require 'mechanize'
+require 'open-uri'
+
+module GScraper
+  #
+  # Returns the GScraper user-agent
+  #
+  def GScraper.user_agent
+    @user_agent
+  end
+
+  #
+  # Sets the GScraper user-agent to the specified _agent_.
+  #
+  def GScraper.user_agent=(agent)
+    @user_agent = agent
+  end
+
+  #
+  # Opens the _uri_ with the given _opts_. The contents of the _uri_ will
+  # be returned.
+  #
+  #   GScraper.open('http://www.hackety.org/')
+  #   GScraper.open('http://tenderlovemaking.com/',
+  #     :user_agent_alias => 'Linux Mozilla')
+  #   GScraper.open('http://www.wired.com/', :user_agent => 'the future')
+  #
+  def GScraper.open(uri,opts={})
+    headers = {}
+
+    if opts[:user_agent_alias]
+      headers['User-Agent'] = WWW::Mechanize::AGENT_ALIASES[opts[:user_agent_alias]]
+    elsif opts[:user_agent]
+      headers['User-Agent'] = opts[:user_agent]
+    elsif GScraper.user_agent
+      headers['User-Agent'] = GScraper.user_agent
+    end
+
+    return Kernel.open(uri,headers)
+  end
+
+  #
+  # Creates a new Mechanize agent with the given _opts_.
+  #
+  #   GScraper.http_agent
+  #   GScraper.http_agent(:user_agent_alias => 'Linux Mozilla')
+  #   GScraper.http_agent(:user_agent => 'wooden pants')
+  #
+  def GScraper.http_agent(opts={})
+    agent = WWW::Mechanize.new
+
+    if opts[:user_agent_alias]
+      agent.user_agent_alias = opts[:user_agent_alias]
+    elsif opts[:user_agent]
+      agent.user_agent = opts[:user_agent]
+    elsif GScraper.user_agent
+      agent.user_agent = GScraper.user_agent
+    end
+
+    return agent
+  end
+end

data/lib/gscraper/licenses.rb

@@ -0,0 +1,56 @@
+module GScraper
+  module Licenses
+    ANY = nil
+
+    ALADDIN = :aladdin
+
+    ARTISTIC = :artistic
+
+    APACHE = :apache
+
+    APPLE = :apple
+
+    BSD = :bsd
+
+    COMMON_PUBLIC = :cpl
+
+    CC_BY = :cc_by
+
+    CC_BY_SA = :cc_by_sa
+
+    CC_BY_ND = :cc_by_nd
+
+    CC_BY_NC = :cc_by_nc_sa
+
+    CC_BY_ND_SA = :cc_by_nd
+
+    CC_BY_NC_SA = :cc_by_nc_sa
+
+    CC_BY_NC_ND = :cc_by_nc_nd
+
+    GPL = :gpl
+
+    LGPL = :lgpl
+
+    HISTORICAL = :disclaimer
+
+    IBM_PUBLIC = :ibm
+
+    LUCENT_PUBLIC = :lucent
+
+    MIT = :mit
+
+    MOZILLA_PUBLI = :mozilla
+
+    NASA_OSA = :nasa
+
+    PYTHON = :python
+
+    Q_PUBLIC = :qpl
+
+    SLEEPYCAT = :sleepycat
+
+    ZOPE_PUBLIC = :zope
+
+  end
+end

data/lib/gscraper/search.rb

@@ -0,0 +1 @@
+require 'gscraper/search/search'

data/lib/gscraper/search/query.rb

@@ -0,0 +1,394 @@
+require 'gscraper/search/result'
+require 'gscraper/extensions/uri'
+require 'gscraper/licenses'
+require 'gscraper/gscraper'
+
+require 'hpricot'
+
+module GScraper
+  module Search
+    class Query
+
+      SEARCH_URL = 'http://www.google.com/search'
+
+      RESULTS_PER_PAGE = 10
+
+      # Results per-page
+      attr_accessor :results_per_page
+
+      # Search query
+      attr_accessor :query
+
+      # Search for results containing the exact phrase
+      attr_accessor :exact_phrase
+
+      # Search for results with the words
+      attr_accessor :with_words
+
+      # Search for results with-out the words
+      attr_accessor :without_words
+
+      # Search for results written in the language
+      attr_accessor :language
+
+      # Search for results from the region
+      attr_accessor :region
+
+      # Search for results in the format
+      attr_accessor :in_format
+
+      # Search for results not in the format
+      attr_accessor :not_in_format
+
+      # Search for results within the past day
+      attr_accessor :within_past_day
+
+      # Search for results within the past week
+      attr_accessor :within_past_week
+
+      # Search for results within the past months
+      attr_accessor :within_past_months
+
+      # Search for results within the past year
+      attr_accessor :within_past_year
+
+      # Search for results containing numbers between the range
+      attr_accessor :numeric_range
+
+      # Search for results where the query ocurrs within the area
+      attr_accessor :occurrs_within
+
+      # Search for results inside the domain
+      attr_accessor :inside_domain
+
+      # Search for results outside the domain
+      attr_accessor :outside_domain
+
+      # Search for results which have the rights
+      attr_accessor :rights
+
+      # Filter the search results
+      attr_accessor :filtered
+
+      # Search for results similar to the page
+      attr_accessor :similar_to
+
+      # Search for results linking to the page
+      attr_accessor :links_to
+
+      #
+      # Creates a new Query object from the given search options. If a
+      # block is given, it will be passed the newly created query object.
+      #
+      #   Query.new(:query => 'ruby', :with_words => 'rspec rails')
+      #
+      #   Query.new(:exact_phrase => 'fluent interfaces') do |q|
+      #     q.within_past_week = true
+      #   end
+      #
+      def initialize(opts={},&block)
+        super()
+
+        @results_per_page = opts[:results_per_page] || RESULTS_PER_PAGE
+
+        @query = opts[:query]
+        @exact_phrase = opts[:exact_phrase]
+        @with_words = opts[:with_words]
+        @without_words = opts[:without_words]
+
+        @language = opts[:language]
+        @region = opts[:region]
+        @in_format = opts[:in_format]
+        @not_in_format = opts[:not_in_format]
+
+        if opts[:within_past_day]
+          @within_past_day = opts[:within_past_day]
+        elsif opts[:within_past_week]
+          @within_past_week = opts[:within_past_week]
+        elsif opts[:within_past_months]
+          @within_past_months = opts[:within_past_months]
+        elsif opts[:within_past_year]
+          @within_past_year = opts[:within_past_year]
+        end
+
+        @numeric_range = opts[:numeric_range]
+        @occurrs_within = opts[:occurrs_within]
+        @inside_domain = opts[:inside_domain]
+        @outside_domain = opts[:outside_domain]
+        @rights = opts[:rights]
+        @filtered = opts[:filtered]
+
+        @similar_to = opts[:similar_to]
+        @links_to = opts[:links_to]
+
+        block.call(self) if block
+      end
+
+      #
+      # Creates a new Query object from the specified URL. If a block is
+      # given, it will be passed the newly created Query object.
+      #
+      #   Query.from_url('http://www.google.com/search?q=ruby+zen)
+      #
+      #   Query.from_url('http://www.google.com/search?q=ruby') do |q|
+      #     q.within_last_month = true
+      #     q.occurrs_within = :title
+      #   end
+      #
+      def self.from_url(url,&block)
+        url = URI.parse(url)
+        opts = {}
+
+        opts[:results_per_page] = url.query_params['num']
+
+        opts[:query] = url.query_params['as_q']
+        opts[:exact_phrase] = url.query_params['as_epq']
+        opts[:with_words] = url.query_params['as_oq']
+        opts[:without_words] = url.query_params['as_eq']
+
+        opts[:language] = url.query_params['lr']
+        opts[:region] = url.query_params['cr']
+
+        case url.query_params['as_ft']
+        when 'i'
+          opts[:in_format] = url.query_params['as_filetype']
+        when 'e'
+          opts[:not_in_format] = url.query_params['as_filetype']
+        end
+
+        case url.query_params['as_qdr']
+        when 'd'
+          opts[:within_past_day] = true
+        when 'w'
+          opts[:within_past_week] = true
+        when 'm'
+          opts[:within_past_months] = 1
+        when 'm2'
+          opts[:within_past_months] = 2
+        when 'm3'
+          opts[:within_past_months] = 3
+        when 'm6'
+          opts[:within_past_months] = 6
+        when 'y'
+          opts[:within_past_year] = true
+        end
+
+        if (url.query_params['as_nlo'] || url.query_params['as_nhi'])
+          opts[:numeric_range] = Range.new(url.query_params['as_nlo'].to_i,url.query_params['as_nhi'].to_i)
+        end
+
+        case url.query_params['as_occt']
+        when 'title'
+          opts[:occurrs_within] = :title
+        when 'body'
+          opts[:occurrs_within] = :body
+        when 'url'
+          opts[:occurrs_within] = :url
+        when 'links'
+          opts[:occurrs_within] = :links
+        end
+
+        case url.query_params['as_dt']
+        when 'i'
+          opts[:inside_domain] = url.query_params['as_sitesearch']
+        when 'e'
+          opts[:outside_domain] = url.query_params['as_sitesearch']
+        end
+
+        case url.query_params['as_rights']
+        when '(cc_publicdomain|cc_attribute|cc_sharealike|cc_noncommercial|cc_nonderived)'
+          opts[:rights] = Licenses::CC_BY_NC_ND
+        when '(cc_publicdomain|cc_attribute|cc_sharealike|cc_nonderived).-(cc_noncommercial)'
+          opts[:rights] = Licenses::CC_BY_SA
+        when '(cc_publicdomain|cc_attribute|cc_sharealike|cc_noncommercial).-(cc_nonderived)'
+          opts[:rights] = Licenses::CC_BY_ND
+        when '(cc_publicdomain|cc_attribute|cc_sharealike).-(cc_noncommercial|cc_nonderived)'
+          opts[:rights] = Licenses::CC_BY
+        end
+
+        if url.query_params[:safe]=='active'
+          opts[:filtered] = true
+        end
+
+        if url.query_params['as_rq']
+          opts[:similar_to] = url.query_params['as_rq']
+        elsif url.query_params['as_lq']
+          opts[:links_to] = url.query_params['as_lq']
+        end
+
+        return self.new(opts,&block)
+      end
+
+      #
+      # Returns the URL that represents the query.
+      #
+      def search_url
+        url = URI.parse(SEARCH_URL)
+
+        if @results_per_page
+          url.query_params['num'] = @results_per_page
+        end
+
+        url.query_params['as_q'] = @query if @query
+        url.query_params['as_epq'] = @exact_phrase if @exact_phrase
+        url.query_params['as_oq'] = @with_words if @with_words
+        url.query_params['as_eq'] = @without_words if @without_words
+
+        url.query_params['lr'] = @language if @language
+        url.query_params['cr'] = @region if @region
+
+        if @in_format
+          url.query_params['as_ft'] = 'i'
+          url.query_params['as_filtetype'] = @in_format
+        elsif @not_in_format
+          url.query_params['as_ft'] = 'e'
+          url.query_params['as_filtetype'] = @not_in_format
+        end
+
+        if @within_past_day
+          url.query_params['as_qdr'] = 'd'
+        elsif @within_past_week
+          url.query_params['as_qdr'] = 'w'
+        elsif @within_past_months
+          case @within_past_months
+          when 1
+            url.query_params['as_qdr'] = 'm'
+          when 2
+            url.query_params['as_qdr'] = 'm2'
+          when 3
+            url.query_params['as_qdr'] = 'm3'
+          when 6
+            url.query_params['as_qdr'] = 'm6'
+          end
+        elsif @within_past_year
+          url.query_params['as_qdr'] = 'y'
+        end
+
+        if @numeric_range
+          url.query_params['as_nlo'] = @numeric_range.begin
+          url.query_params['as_nhi'] = @numeric_range.end
+        end
+
+        case @occurrs_within
+        when :title, 'title'
+          url.query_params['as_occt'] = 'title'
+        when :body, 'body'
+          url.query_params['as_occt'] = 'body'
+        when :url, 'url'
+          url.query_params['as_occt'] = 'url'
+        when :links, 'links'
+          url.query_params['as_occt'] = 'links'
+        end
+
+        if @inside_domain
+          url.query_params['as_dt'] = 'i'
+          url.query_params['as_sitesearch'] = @inside_domain
+        elsif @outside_domain
+          url.query_params['as_dt'] = 'e'
+          url.query_params['as_sitesearch'] = @outside_domain
+        end
+
+        case @rights
+        when Licenses::CC_BY_NC_ND
+          url.query_params['as_rights'] = '(cc_publicdomain|cc_attribute|cc_sharealike|cc_noncommercial|cc_nonderived)'
+        when Licenses::CC_BY_SA
+          url.query_params['as_rights'] = '(cc_publicdomain|cc_attribute|cc_sharealike|cc_nonderived).-(cc_noncommercial)'
+        when Licenses::CC_BY_ND
+          url.query_params['as_rights'] = '(cc_publicdomain|cc_attribute|cc_sharealike|cc_noncommercial).-(cc_nonderived)'
+        when Licenses::CC_BY
+          url.query_params['as_rights'] = '(cc_publicdomain|cc_attribute|cc_sharealike).-(cc_noncommercial|cc_nonderived)'
+        end
+
+        url.query_params['safe'] = true if @filtered
+
+        if @similar_to
+          url.query_params['as_rq'] = @similar_to
+        elsif @links_to
+          url.query_params['as_lq'] = @links_to
+        end
+
+        return url
+      end
+
+      #
+      # Returns the URL that represents the query at the specific
+      # _page_index_.
+      #
+      def page_url(page_index)
+        url = search_url
+
+        url.query_params['start'] = page_index_offset(page_index)
+        url.query_params['sa'] = 'N'
+
+        return url
+      end
+
+      #
+      # Returns an array of Result objects at the specified _page_index_.
+      # If _opts_ are given, they will be used in accessing the SEARCH_URL.
+      #
+      def page(page_index,opts={})
+        results = []
+        doc = Hpricot(GScraper.open(page_url(page_index),opts))
+
+        doc.search('//div.g').each_with_index do |result,index|
+          rank = page_index_offset(page_index) + (index + 1)
+          title = result.search('//h2.r').first.inner_text
+          url = result.search('//h2.r/a').first.get_attribute('href')
+          # TODO: exclude URL and Links from summary text
+          summary = result.search('//td.j').first.inner_text
+
+          # TODO: scrape Cached and Similar links
+
+          results << Result.new(rank,title,url,summary)
+        end
+
+        return results
+      end
+
+      #
+      # Returns the results on the first page. If _opts_ are given, they
+      # will be used in accessing the SEARCH_URL.
+      #
+      def first_page(opts={})
+        page(1,opts)
+      end
+
+      #
+      # Iterates over the results at the specified _page_index_, passing
+      # each to the given _block_. If _opts_ are given they will be used
+      # in accessing the SEARCH_URL.
+      #
+      #   query.each_on_page(2) do |result|
+      #     puts result.title
+      #   end
+      #
+      def each_on_page(page_index,opts={},&block)
+        page(page_index,opts).each(&block)
+      end
+
+      #
+      # Iterates over the results on the first page, passing
+      # each to the given _block_. If _opts_ are given, they will be used
+      # in accessing the SEARCH_URL.
+      #
+      #   query.each_on_first_page do |result|
+      #     puts result.url
+      #   end
+      #
+      def each_on_first_page(opts={},&block)
+        each_on_page(1,opts,&block)
+      end
+
+      protected
+
+      #
+      # Returns the rank offset for the specified _page_index_.
+      #
+      def page_index_offset(page_index)
+        (page_index.to_i - 1) * @result_per_page.to_i
+      end
+
+    end
+  end
+end

data/lib/gscraper/search/result.rb

@@ -0,0 +1,37 @@
+module GScraper
+  module Search
+    class Result
+
+      # Rank of the result page
+      attr_reader :rank
+
+      # Title of the result page
+      attr_reader :title
+
+      # URL of the result page
+      attr_reader :url
+
+      # Summary from the result page
+      attr_reader :summary
+
+      #
+      # Creates a new Result object with the given _rank_, _title_
+      # _summary_, _url_ and _size_.
+      #
+      def initialize(rank,title,url,summary)
+        @rank = rank
+        @title = title
+        @url = url
+        @summary = summary
+      end
+
+      #
+      # Returns a string containing the result's title.
+      #
+      def to_s
+        @title.to_s
+      end
+
+    end
+  end
+end

data/lib/gscraper/search/search.rb

@@ -0,0 +1,33 @@
+require 'gscraper/search/query'
+
+module GScraper
+  module Search
+    #
+    # Returns a new Query object with the given _opts_. See Query.new.
+    #
+    #   Search.query(:query => 'ruby', :with_words => 'rspec rails')
+    #
+    #   Search.query(:exact_phrase => 'fluent interfaces') do |q|
+    #     q.within_past_week = true
+    #   end
+    #
+    def Search.query(opts={},&block)
+      Query.new(opts,&block)
+    end
+
+    #
+    # Returns the Query object that represents the specified _url_.
+    # See Query.from_url.
+    #
+    #   Search.query_from_url('http://www.google.com/search?q=ruby+zen)
+    #
+    #   Search.query_from_url('http://www.google.com/search?q=ruby') do |q|
+    #     q.within_last_month = true
+    #     q.occurrs_within = :title
+    #   end
+    #
+    def Search.query_from_url(url,&block)
+      Query.from_url(url,&block)
+    end
+  end
+end

data/test/search/query_from_url.rb

@@ -0,0 +1,50 @@
+require 'test/unit'
+require 'gscraper/search/query'
+
+class QueryFromURL < Test::Unit::TestCase
+
+  include GScraper
+
+  QUERY_URL = 'http://www.google.com/search?as_q=test&hl=en&num=20&btnG=Google+Search&as_epq=what+if&as_oq=dog&as_eq=haha&lr=&cr=&as_ft=i&as_filetype=&as_qdr=w&as_nlo=&as_nhi=&as_occt=body&as_dt=i&as_sitesearch=&as_rights=&safe=images'
+
+  def setup
+    @query = Search::Query.from_url(QUERY_URL)
+  end
+
+  def teardown
+    @query = nil
+  end
+
+  def test_query
+    assert_equal @query.query, 'test'
+  end
+
+  def test_exact_phrase
+    assert_equal @query.exact_phrase, 'what+if'
+  end
+
+  def test_with_words
+    assert_equal @query.with_words, 'dog'
+  end
+
+  def test_without_words
+    assert_equal @query.without_words, 'haha'
+  end
+
+  def test_within_past_week
+    assert_equal @query.within_past_week, true
+  end
+
+  def test_occurrs_within
+    assert_equal @query.occurrs_within, :body
+  end
+
+  def test_similar_to
+    assert_nil @query.similar_to
+  end
+
+  def test_links_to
+    assert_nil @query.links_to
+  end
+
+end

data/test/test_gscraper.rb

@@ -0,0 +1,4 @@
+$LOAD_PATH.unshift(File.expand_path(File.join(File.dirname(__FILE__),'..','lib')))
+
+require 'test/unit'
+require 'search/query_from_url'

metadata

@@ -0,0 +1,74 @@
+--- !ruby/object:Gem::Specification 
+rubygems_version: 0.9.4
+specification_version: 1
+name: gscraper
+version: !ruby/object:Gem::Version 
+  version: 0.1.0
+date: 2007-12-21 00:00:00 -08:00
+summary: A ruby web-scraping interface to various Google Services
+require_paths: 
+- lib
+email: postmodern.mod3@gmail.com
+homepage: "    by Postmodern Modulus III"
+rubyforge_project: gscraper
+description: "== FEATURES/PROBLEMS:  * Supports the Google Search service. * Provides HTTP access with custom User-Agent strings.  == REQUIREMENTS:  * Hpricot * Mechanize  == INSTALL:"
+autorequire: 
+default_executable: 
+bindir: bin
+has_rdoc: true
+required_ruby_version: !ruby/object:Gem::Version::Requirement 
+  requirements: 
+  - - ">"
+    - !ruby/object:Gem::Version 
+      version: 0.0.0
+  version: 
+platform: ruby
+signing_key: 
+cert_chain: 
+post_install_message: 
+authors: 
+- Postmodern Modulus III
+files: 
+- History.txt
+- LICENSE.txt
+- Manifest.txt
+- README.txt
+- Rakefile
+- lib/gscraper.rb
+- lib/gscraper/gscraper.rb
+- lib/gscraper/extensions/uri/http.rb
+- lib/gscraper/extensions/uri.rb
+- lib/gscraper/extensions.rb
+- lib/gscraper/licenses.rb
+- lib/gscraper/search/result.rb
+- lib/gscraper/search/query.rb
+- lib/gscraper/search/search.rb
+- lib/gscraper/search.rb
+- test/test_gscraper.rb
+- test/search/query_from_url.rb
+test_files: 
+- test/test_gscraper.rb
+rdoc_options: 
+- --main
+- README.txt
+extra_rdoc_files: 
+- History.txt
+- LICENSE.txt
+- Manifest.txt
+- README.txt
+executables: []
+
+extensions: []
+
+requirements: []
+
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: hoe
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Version::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: 1.3.0
+    version: