gscraper 0.1.6 → 0.1.7

data/History.txt

@@ -1,3 +1,11 @@
+== 0.1.7 / 2008-04-28
+
+* Added support for specifing Search modifiers.
+
+  Search.query(:filetype => :xls)
+
+* Added the Search::Result#page method.
+
 == 0.1.6 / 2008-03-15
 
 * Renamed GScraper.http_agent to GScraper.web_agent.

data/README.txt

@@ -66,7 +66,7 @@ GScraper is a web-scraping interface to various Google Services.
     page = q.page(2)
 
     page.urls # => [...]
-    pagesummaries # => [...]
+    page.summaries # => [...]
     page.ranks_of { |result| result.url =~ /^https/ } # => [...]
     page.titles_of { |result| result.summary =~ /password/ } # => [...]
     page.cached_pages # => [...]
@@ -127,7 +127,7 @@ GScraper is a web-scraping interface to various Google Services.
 
 The MIT License
 
-Copyright (c) 2007 Hal Brodigan
+Copyright (c) 2007-2008 Hal Brodigan
 
 Permission is hereby granted, free of charge, to any person obtaining
 a copy of this software and associated documentation files (the

data/lib/gscraper/search/query.rb

@@ -29,6 +29,39 @@ module GScraper
       # Search query
       attr_accessor :query
 
+      # Search 'link' modifier
+      attr_accessor :link
+
+      # Search 'related' modifier
+      attr_accessor :related
+
+      # Search 'info' modifier
+      attr_accessor :info
+
+      # Search 'site' modifier
+      attr_accessor :site
+
+      # Search 'filetype' modifier
+      attr_accessor :filetype
+
+      # Search 'allintitle' modifier
+      attr_accessor :allintitle
+
+      # Search 'intitle' modifier
+      attr_accessor :intitle
+
+      # Search 'allinurl' modifier
+      attr_accessor :allinurl
+
+      # Search 'inurl' modifier
+      attr_accessor :inurl
+
+      # Search 'allintext' modifier
+      attr_accessor :allintext
+
+      # Search 'intext' modifier
+      attr_accessor :intext
+
       # Search for results containing the exact phrase
       attr_accessor :exact_phrase
 
@@ -100,6 +133,20 @@ module GScraper
         @results_per_page = (options[:results_per_page] || RESULTS_PER_PAGE)
 
         @query = options[:query]
+
+        @link = options[:link]
+        @related = options[:related]
+        @info = options[:info]
+        @site = options[:site]
+        @filetype = options[:filetype]
+
+        @allintitle = options[:allintitle]
+        @intitle = options[:intitle]
+        @allinurl = options[:allinurl]
+        @inurl = options[:inurl]
+        @allintext = options[:allintext]
+        @intext = options[:intext]
+
         @exact_phrase = options[:exact_phrase]
         @with_words = options[:with_words]
         @without_words = options[:without_words]
@@ -248,18 +295,55 @@ module GScraper
       #
       def search_url
         url = URI(SEARCH_URL)
+        query_expr = []
+
+        set_param = lambda { |param,value|
+          url.query_params[param.to_s] = value if value
+        }
+
+        append_modifier = lambda { |name|
+          modifier = instance_variable_get("@#{name}")
+
+          query_expr << "#{name}:#{modifier}" if modifier
+        }
+
+        join_ops = lambda { |name|
+          ops = instance_variable_get("@#{name}")
+
+          if ops.kind_of?(Array)
+            query_expr << "#{name}:#{ops.join(' ')}"
+          elsif ops
+            query_expr << "#{name}:#{ops}"
+          end
+        }
+
+        set_param.call('num',@results_per_page)
+
+        query_expr << @query if @query
+
+        append_modifier.call(:link)
+        append_modifier.call(:related)
+        append_modifier.call(:info)
+        append_modifier.call(:site)
+        append_modifier.call(:filetype)
+
+        join_ops.call(:allintitle)
+        append_modifier.call(:intitle)
+        join_ops.call(:allinurl)
+        append_modifier.call(:inurl)
+        join_ops.call(:allintext)
+        append_modifier.call(:intext)
 
-        if @results_per_page
-          url.query_params['num'] = @results_per_page
+        unless query_expr.empty?
+          url.query_params['as_q'] = query_expr.join(' ')
         end
 
-        url.query_params['as_q'] = @query if @query
-        url.query_params['as_epq'] = @exact_phrase if @exact_phrase
-        url.query_params['as_oq'] = @with_words if @with_words
-        url.query_params['as_eq'] = @without_words if @without_words
+        set_param.call('as_epq',@exact_phrase)
+        set_param.call('as_oq',@with_words)
+        set_param.call('as_eq',@without_words)
 
-        url.query_params['lr'] = @language if @language
-        url.query_params['cr'] = @region if @region
+        set_param.call('lr',@language)
+        set_param.call('cr',@region)
 
         if @in_format
           url.query_params['as_ft'] = 'i'

data/lib/gscraper/search/result.rb

@@ -38,6 +38,14 @@ module GScraper
         @similar_url = similar_url
       end
 
+      #
+      # Fetches the page of the result. If a _block_ is given it will be
+      # passed the page.
+      #
+      def page(&block)
+        get_page(@url,&block)
+      end
+
       #
       # Create a new Query for results that are similar to the Result. If
       # a _block_ is given, it will be passed the newly created Query

data/lib/gscraper/version.rb

@@ -1,3 +1,3 @@
 module GScraper
-  VERSION = '0.1.6'
+  VERSION = '0.1.7'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: gscraper
 version: !ruby/object:Gem::Version 
-  version: 0.1.6
+  version: 0.1.7
 platform: ruby
 authors: 
 - Postmodern Modulus III
@@ -9,7 +9,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2008-03-15 00:00:00 -07:00
+date: 2008-04-28 00:00:00 -07:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -99,7 +99,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
 requirements: []
 
 rubyforge_project: gscraper
-rubygems_version: 1.0.1
+rubygems_version: 1.1.1
 signing_key: 
 specification_version: 2
 summary: A ruby web-scraping interface to various Google Services