gscraper 0.3.0 → 0.4.0

data/lib/gscraper/has_pages.rb

@@ -1,7 +1,7 @@
 #
 # GScraper - A web-scraping interface to various Google Services.
 #
-# Copyright (c) 2007-2009 Hal Brodigan (postmodern.mod3 at gmail.com)
+# Copyright (c) 2007-2012 Hal Brodigan (postmodern.mod3 at gmail.com)
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -18,8 +18,6 @@
 # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
 #
 
-require 'enumerator'
-
 module GScraper
   module HasPages
     include Enumerable

data/lib/gscraper/hosts.rb

@@ -0,0 +1,158 @@
+#
+# GScraper - A web-scraping interface to various Google Services.
+#
+# Copyright (c) 2007-2012 Hal Brodigan (postmodern.mod3 at gmail.com)
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+#
+
+module GScraper
+  #
+  # @api semipublic
+  #
+  # @since 0.4.0
+  #
+  module Hosts
+    # List of all google domain-names.
+    DOMAINS = %w[
+      google.com
+      google.de
+      google.at
+      google.pl
+      google.fr
+      google.nl
+      google.it
+      google.com.tr
+      google.es
+      google.ch
+      google.be
+      google.gr
+      google.com.br
+      google.lu
+      google.fi
+      google.pt
+      google.hu
+      google.hr
+      google.bg
+      google.com.mx
+      google.si
+      google.sk
+      google.ro
+      google.ca
+      google.co.uk
+      google.cl
+      google.com.ar
+      google.se
+      google.cz
+      google.dk
+      google.co.th
+      google.com.co
+      google.lt
+      google.co.id
+      google.co.in
+      google.co.il
+      google.com.eg
+      google.cn
+      google.co.ve
+      google.ru
+      google.co.jp
+      google.com.pe
+      google.com.au
+      google.co.ma
+      google.co.za
+      google.com.ph
+      google.com.sa
+      google.ie
+      google.co.kr
+      google.no
+      google.com.ec
+      google.com.vn
+      google.lv
+      google.com.mt
+      google.com.uy
+      google.ae
+      google.ba
+      google.co.nz
+      google.com.ua
+      google.co.cr
+      google.ee
+      google.com.do
+      google.com.tw
+      google.com.hk
+      google.com.my
+      google.com.sv
+      google.com.pr
+      google.lk
+      google.com.gt
+      google.com.bd
+      google.com.pk
+      google.is
+      google.li
+      google.com.bh
+      google.com.ni
+      google.com.py
+      google.com.ng
+      google.com.bo
+      google.co.ke
+      google.hn
+      google.com.sg
+      google.mu
+      google.ci
+      google.jo
+      google.nu
+      google.com.jm
+      google.com.ly
+      google.co.yu
+      google.tt
+      google.com.kh
+      google.ge
+      google.com.na
+      google.com.et
+      google.sm
+      google.cd
+      google.gm
+      google.com.qa
+      google.dj
+      google.com.cu
+      google.com.pa
+      google.gp
+      google.az
+      google.as
+      google.pl
+      google.mn
+      google.ht
+      google.md
+      google.am
+      google.sn
+      google.je
+      google.com.bn
+      google.com.ai
+      google.co.zm
+      google.ma
+      google.rw
+      google.co.ug
+      google.com.vc
+      google.at
+      google.com.gi
+      google.to
+      google.com.om
+      google.kz
+      google.co.uz
+    ]
+
+    # The primary domain
+    PRIMARY_DOMAIN = DOMAINS.first
+  end
+end

data/lib/gscraper/languages.rb

@@ -0,0 +1,110 @@
+#
+# GScraper - A web-scraping interface to various Google Services.
+#
+# Copyright (c) 2007-2012 Hal Brodigan (postmodern.mod3 at gmail.com)
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+#
+
+module GScraper
+  #
+  # @api semipublic
+  #
+  # @since 0.3.0
+  #
+  module Languages
+    # The list of language names
+    NAMES = %w[
+      af
+      ar
+      be
+      bg
+      ca
+      cs
+      da
+      de
+      el
+      en
+      eo
+      es
+      et
+      fa
+      fi
+      fr
+      hi
+      hr
+      hu
+      hy
+      id
+      is
+      it
+      iw
+      ja
+      ko
+      lt
+      lv
+      nl
+      no
+      pl
+      pt
+      ro
+      ru
+      sk
+      sl
+      sr
+      sv
+      sw
+      th
+      tl
+      tr
+      uk
+      vi
+      zh-CN
+      zh-TW
+    ]
+
+    #
+    # Looks up the language for the given locale.
+    #
+    # @param [String] locale
+    #   A locale.
+    #
+    # @return [String]
+    #   The language used by the locale.
+    #
+    def Languages.find(locale)
+      if locale =~ /^zh_CN/
+        'zh-CN'
+      elsif locale =~ /^zh_TW/
+        'zh-TW'
+      else
+        if (match = locale.match(/^([^_@]+)([_@].+)?$/))
+          match[1] if (match[1] && NAMES.include?(match[1]))
+        end
+      end
+    end
+
+    #
+    # Determines the native language.
+    #
+    # @return [String]
+    #   The native language.
+    #
+    def Languages.native
+      language = ENV['LANG'] || 'en'
+      Languages.find(language)
+    end
+  end
+end

data/lib/gscraper/licenses.rb

@@ -1,7 +1,7 @@
 #
 # GScraper - A web-scraping interface to various Google Services.
 #
-# Copyright (c) 2007-2009 Hal Brodigan (postmodern.mod3 at gmail.com)
+# Copyright (c) 2007-2012 Hal Brodigan (postmodern.mod3 at gmail.com)
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -19,6 +19,9 @@
 #
 
 module GScraper
+  #
+  # @api semipublic
+  #
   module Licenses
     # Any desired license
     ANY = nil

data/lib/gscraper/page.rb

@@ -1,7 +1,7 @@
 #
 # GScraper - A web-scraping interface to various Google Services.
 #
-# Copyright (c) 2007-2009 Hal Brodigan (postmodern.mod3 at gmail.com)
+# Copyright (c) 2007-2012 Hal Brodigan (postmodern.mod3 at gmail.com)
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -18,8 +18,6 @@
 # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
 #
 
-require 'enumerator'
-
 module GScraper
   class Page < Array
 

data/lib/gscraper/search.rb

@@ -1,7 +1,7 @@
 #
 # GScraper - A web-scraping interface to various Google Services.
 #
-# Copyright (c) 2007-2009 Hal Brodigan (postmodern.mod3 at gmail.com)
+# Copyright (c) 2007-2012 Hal Brodigan (postmodern.mod3 at gmail.com)
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by

data/lib/gscraper/search/ajax_query.rb

@@ -1,7 +1,7 @@
 #
 # GScraper - A web-scraping interface to various Google Services.
 #
-# Copyright (c) 2007-2009 Hal Brodigan (postmodern.mod3 at gmail.com)
+# Copyright (c) 2007-2012 Hal Brodigan (postmodern.mod3 at gmail.com)
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -21,11 +21,11 @@
 require 'gscraper/search/result'
 require 'gscraper/search/page'
 require 'gscraper/search/query'
-require 'gscraper/extensions/uri'
 require 'gscraper/has_pages'
 require 'gscraper/gscraper'
 
 require 'json'
+require 'uri/query_params'
 require 'nokogiri'
 
 module GScraper
@@ -40,14 +40,11 @@ module GScraper
       # Maximum results per-page
       RESULTS_PER_PAGE = 8
 
-      # AJAX API host
-      API_HOST = 'www.google.com'
+      # AJAX API Path
+      PATH = '/uds/GwebSearch'
 
-      # AJAX API URL
-      API_URL = "http://#{API_HOST}/uds/GwebSearch?callback=google.search.WebSearch.RawCompletion&context=0&lstkp=0&rsz=large"
-
-      # Default language
-      DEFAULT_LANGUAGE = 'en'
+      # AJAX API Query string
+      QUERY = 'callback=google.search.WebSearch.RawCompletion&context=0&lstkp=0&rsz=large'
 
       # Default signature
       DEFAULT_SIG = '582c1116317355adf613a6a843f19ece'
@@ -58,9 +55,6 @@ module GScraper
       # Default version
       DEFAULT_VERSION = '1.0'
 
-      # The search language
-      attr_accessor :language
-
       # The search signature
       attr_accessor :sig
 
@@ -76,13 +70,16 @@ module GScraper
       # @param [Hash] options
       #   Query options.
       #
-      # @option options [Symbol] :language (:en)
+      # @option options [String] :search_host (www.google.com)
+      #   The host to submit queries to.
+      #
+      # @option options [String, Symbol] :language (Languages.native)
       #   The search language.
       #
       # @option options [String] :sig ('582c1116317355adf613a6a843f19ece')
       #   The search signature.
       #
-      # @option options [Symbol] :key (:notsupplied)
+      # @option options [String, Symbol] :key ('notsupplied')
       #   The search key.
       #
       # @option options [Float] :version (1.0)
@@ -97,11 +94,9 @@ module GScraper
       def initialize(options={},&block)
         @agent = GScraper.web_agent(options)
 
-        @language = (options[:language] || DEFAULT_LANGUAGE)
-
-        @sig = (options[:sig] || DEFAULT_SIG)
-        @key = (options[:key] || DEFAULT_KEY)
-        @version = (options[:version] || DEFAULT_VERSION)
+        @sig     = options.fetch(:sig,DEFAULT_SIG)
+        @key     = options.fetch(:key,DEFAULT_KEY)
+        @version = options.fetch(:version,DEFAULT_VERSION)
 
         super(options,&block)
       end
@@ -130,13 +125,13 @@ module GScraper
         url = URI(url.to_s)
 
         options[:language] = url.query_params['hl']
-        options[:query] = url.query_params['q']
+        options[:query]    = url.query_params['q']
 
-        options[:sig] = url.query_params['sig']
-        options[:key] = url.query_params['key']
+        options[:sig]     = url.query_params['sig']
+        options[:key]     = url.query_params['key']
         options[:version] = url.query_params['v']
 
-        return self.new(options,&block)
+        return AJAXQuery.new(options,&block)
       end
 
       #
@@ -158,14 +153,18 @@ module GScraper
       #   The URL for the query.
       #
       def search_url
-        search_url = URI(API_URL)
+        search_url = URI::HTTP.build(
+          :host  => search_host,
+          :path  => PATH,
+          :query => QUERY
+        )
 
-        search_url.query_params['hl'] = @language
+        search_url.query_params['hl']  = @language
         search_url.query_params['gss'] = '.com'
-        search_url.query_params['q'] = expression
+        search_url.query_params['q']   = expression
         search_url.query_params['sig'] = @sig
         search_url.query_params['key'] = @key
-        search_url.query_params['v'] = @version
+        search_url.query_params['v']   = @version
 
         return search_url
       end
@@ -207,15 +206,15 @@ module GScraper
 
           if (hash.kind_of?(Hash) && hash['results'])
             hash['results'].each_with_index do |result,index|
-              rank = rank_offset + (index + 1)
+              rank  = rank_offset + (index + 1)
               title = Nokogiri::HTML(result['title']).inner_text
-              url = URI(URI.escape(result['unescapedUrl']))
+              url   = URI(URI.escape(result['unescapedUrl']))
 
-              unless result['content'].empty?
-                summary = Nokogiri::HTML(result['content']).inner_text
-              else
-                summary = ''
-              end
+              summary = unless result['content'].empty?
+                          Nokogiri::HTML(result['content']).inner_text
+                        else
+                          ''
+                        end
 
               cached_url = URI(result['cacheUrl'])