google_custom_search_api 1.0.0 → 2.0.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 228c6676867465f4a11356a89dd36130f926f38d
+  data.tar.gz: 9aca83a6a4cc013be2d2bfac1c1928add7104a88
+SHA512:
+  metadata.gz: 76fc199878f446ef2ba231c8ca3a3a29034debe1ea9943f2a1d30e00c99984926bcddc2f230e6b088af55d38053178f2e075d77be69b5b87218f22d4ebedf56d
+  data.tar.gz: 825d19ec8e359ea688cc72ff321eae2c791b7f043a801e081b6867ae45df914a8655cf1b60ca6a2da3111687933da6212309ead48f8c13b3c06352f1d64e2089

data/Gemfile.lock

@@ -1,32 +1,42 @@
 PATH
   remote: .
   specs:
-    google_custom_search_api (0.0.4)
+    google_custom_search_api (2.0.0)
+      addressable
       httparty
 
 GEM
   remote: http://rubygems.org/
   specs:
-    addressable (2.2.7)
-    crack (0.3.1)
-    diff-lcs (1.1.3)
-    httparty (0.8.1)
-      multi_json
-      multi_xml
-    json (1.6.5)
-    multi_json (1.3.4)
-    multi_xml (0.4.2)
-    rspec (2.9.0)
-      rspec-core (~> 2.9.0)
-      rspec-expectations (~> 2.9.0)
-      rspec-mocks (~> 2.9.0)
-    rspec-core (2.9.0)
-    rspec-expectations (2.9.0)
-      diff-lcs (~> 1.1.3)
-    rspec-mocks (2.9.0)
-    webmock (1.8.3)
-      addressable (>= 2.2.7)
-      crack (>= 0.1.7)
+    addressable (2.3.8)
+    crack (0.4.3)
+      safe_yaml (~> 1.0.0)
+    diff-lcs (1.2.5)
+    hashdiff (0.2.3)
+    httparty (0.13.7)
+      json (~> 1.8)
+      multi_xml (>= 0.5.2)
+    json (1.8.3)
+    multi_xml (0.5.5)
+    rspec (3.4.0)
+      rspec-core (~> 3.4.0)
+      rspec-expectations (~> 3.4.0)
+      rspec-mocks (~> 3.4.0)
+    rspec-core (3.4.1)
+      rspec-support (~> 3.4.0)
+    rspec-expectations (3.4.0)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.4.0)
+    rspec-mocks (3.4.0)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.4.0)
+    rspec-support (3.4.1)
+    safe_yaml (1.0.4)
+    vcr (3.0.1)
+    webmock (1.22.5)
+      addressable (< 2.4.0)
+      crack (>= 0.3.2)
+      hashdiff
 
 PLATFORMS
   ruby
@@ -35,4 +45,8 @@ DEPENDENCIES
   google_custom_search_api!
   json
   rspec
+  vcr
   webmock
+
+BUNDLED WITH
+   1.10.6

data/README.md

@@ -22,19 +22,30 @@ You need to configure ```GOOGLE_SEARCH_CX``` and ```GOOGLE_API_KEY``` to ```conf
   GOOGLE_API_KEY = "..."
   GOOGLE_SEARCH_CX = "..."
 ```
-You can get your ```GOOGLE_API_KEY``` from https://code.google.com/apis/console/b/0/?pli=1 - There are many choices - Simple API Access is probably what you want.  There are more elaborate authorization schemes available for Google services but those aren't currently implemented.
 
-You can get your ```GOOGLE_SEARCH_CX``` from http://www.google.com/cse/  Either create a custom engine or follow ```manage your existing search engines``` and go to your cse's Control panel.  ```GOOGLE_SEARCH_CX``` == ```Search engine unique ID```
+Google's API management is confusing at best. At the time of this writing you codes like so:
 
-### Searching the web, not just your site, with CSE
+### GOOGLE_API_KEY
 
-Google CSE was set up so search specific sites.  To search the entire web simply go to http://www.google.com/cse/, find your CSE, go to it's control panel.
+* Go to [Google Projects](https://console.developers.google.com/project)
+* Create a project, open it
+* Under `Explore other services` choose `Enable APIs and get credentials like keys`
+* Search for `custom search` and click on it
+* In the left column click on `Credentials`
+* Under `API keys` grab your key. This is your `GOOGLE_API_KEY`
 
-* in ```Basics``` under ```Search Preferences``` choose ```Search the entire web but emphasize included sites.```
-* in ```Sites``` add ```www.google.com```
+### GOOGLE_SEARCH_CX
+
+* Go to [Google CSE](https://cse.google.com/cse)
+* Create a search engine and click on it
+* Under `Setup > Tabs > Basic` find `Details` and click `Search engine ID`
+* This is your GOOGLE_SEARCH_CX
+* Make sure to add a site under `Sites to search`
 
 ## Use
 
+### Search
+
 To perform a search:
 
 ```
@@ -58,49 +69,189 @@ or
   end
 ```
 
-You can get all ten pages at once by doing:
+### Paging
+
+Google only returns 10 results at a time and a maximum of 100 results. The easiest way to page through results if to use `:page`. Paging is 1 based (1-10). The default page is 1
 
 ```
-  results = GoogleCustomSearchApi.search_and_return_all_results(query, opts) 
-  results.size == 10
-  results.collect {|r| r.items.size }.sum == 100 #if there were 100 results
+  results = GoogleCustomerSearchApi.search("poker", page: 2)
+  results.pages == 10
+  results.current_page == 2
+  results.next_page == 3
+  results.previous_page == 1
+
+  results = GoogleCustomerSearchApi.search("poker", page: 1)
+  results.pages == 10
+  results.current_page == 1
+  results.next_page == 2
+  results.previous_page == nil
+
+  results = GoogleCustomerSearchApi.search("poker", page: 10)
+  results.pages == 10
+  results.current_page == 10
+  results.next_page == nil
+  results.previous_page == 9
 ```
 
-search_and_return_all_results also yields results as it goes:
+You can also use `:start` - which can be any number between 1 and 99. The `:page` helpers won't be accurate with `:start`
+
+Example: get results 13-23
 
 ```
-  GoogleCustomSearchApi.search_and_return_all_results(query, opts) do |results|
-    results.items.size == 10
-  end
+  results = GoogleCustomerSearchApi.search('poker', start: 13)
 ```
 
 See [Custom Search](http://code.google.com/apis/customsearch/v1/using_rest.html) documentation for an explanation of all fields available.
 
-### Paging
+### Search and return all results
+
+This method isn't so useful because it's pretty slow (do to fetching up to 10 pages from Google). Helpful for testing sometimes.
+
+```
+  results = search_and_return_all_results('poker')
+  results.first.items.size # == 10
+  
+  search_and_resturn_all_results('poker') do |results|
+    results.items.size # == 10  10 times
+  end
+  
+  search_and_return_all_results(
+    '"California cult winery known for its Rhône"') do |results|
+    results.items.size # == 3  1 time
+  end
+```
+
+### Errors
+
+Custom Search only returns a maximum of 100 results so - if you try something like 
+
+```
+  results = GoogleCustomSearchApi.search('poker', start: 101)
+```
+You get error and empty items. 
+
+```
+	{
+	  "error"=> {
+	    "errors"=> [
+	      {
+	        "domain"=>"global", 
+	         "reason"=>"invalid", 
+	         "message"=>"Invalid Value"
+	      }
+	    ], 
+	    "code"=>400, 
+	    "message"=>"Invalid Value"
+	  }, 
+	  "items"=>[]
+	}
+```
+
+So check for:
+
+```
+  if results.try(:error) || results.items.empty?
+```
+### Rails example
+
+In **Gemfile**
+
+```
+gem "google_custom_search_api"
+```
+
+In **config/initializers/google_search.rb**
 
-By default CSE returns a maximum of 10 results at a time, you can't get more results without paging. BTW if you want fewer results just pass in the :num => 1-10 option when searching.
+```
+GOOGLE_API_KEY = '...'
+GOOGLE_SEARCH_CX = '...'
+```
 
-To do paging we pass in the :start option.  Example:
+In **config/routes.rb**
 
 ```
-  results = GoogleCustomSearchApi.search("poker", :start => 1)
+  get '/search' => 'search#index'
 ```
 
-The maximum number of pages CSE allows is 10 - or 100 results in total.  To walk through the pages you can use :start => 1, :start => 11, etc. Or you can use the results to find the next value, like so:
+In **app/controllers/search_controller.rb** you'd have something like this:
 
 ```
-  start = 1
-  begin
-    results = GoogleCustomSearchApi.search("poker",:start => start)
-    if results.queries.keys.include?("nextPage")
-      start = results.queries.nextPage.first.startIndex
-    else
-      start = nil
-    end
-  end while start.nil? == false
+class SearchController < ApplicationController
+  def index
+    if params[:q]
+      page = params[:page] || 1
+      @results = GoogleCustomSearchApi.search(params[:q],
+                                              page: page)
+	end
+  end
+end
 ```
 
-If you just want all results you can use the method ```search_and_return_all_results(query, opts = {})``` works just like the normal search but iterates through all available results and puts them in an array.
+And a simple view might look like this **app/search/index.html.erb** (this is using bootstrap styling)
+
+```
+<section class='search-section'>
+  <div class='text-center titles-with-yellow'>
+    <h1>Search/h1>
+  </div>
+  <div class='container'>
+    <div class='text-center search-bar'>
+      <%= form_tag search_path, method: :get  do %>
+        <div class="inner-addon right-addon">
+          <i class="glyphicon glyphicon-search"></i>
+          <%= text_field_tag :q, params[:q], class: 'form-control' %>
+        </div>
+      <% end %>
+    </div>
+  </div>
+
+  <% if @results && !@results.items.empty? %>
+    <div class='container'>
+      <% @results.items.each do |item| %>
+        <div class='row'>
+          <h4><%= link_to item.htmlTitle.html_safe, item.link %></h4>
+          <div>
+            <% if item['pagemap'] &&
+                  item['pagemap']['cse_thumbnail'] &&
+                  img = item.pagemap.cse_thumbnail.first %>
+              <div class='col-sm-2'>
+                <%= image_tag(img.src, width: '200px') %>
+              </div>
+              <div class='col-sm-10'>
+                <%= item.htmlSnippet.html_safe %>
+              </div>
+            <% else %>
+              <%= item.htmlSnippet.html_safe %>
+            <% end %>
+          </div>
+        </div>
+      <% end %>
+    </div>
+    <div class='container search-prev-next'>
+      <div class='row text-center'>
+        <% if @results.previous_page %>
+          <%= link_to '<< Previous',
+            search_path(q: params[:q], page: @results.previous_page),
+            class: 'btn' %>
+        <% end %>
+        <% @results.pages.times do |i| %>
+          <%= link_to i + 1,
+            search_path(q: params[:q], page: i+1),
+            class: 'btn btn-page' %>
+        <% end %>
+        <% if @results.next_page %>
+          <%= link_to 'Next >>',
+            search_path(q: params[:q],
+                        page: @results.next_page),
+            class: 'btn' %>
+        <% end %>
+      </div>
+    </div>
+  <% else %>
+    <h4>No results</h4>
+  <% end %>
+</section>
+```
 
 ### Encoding issues
 
@@ -131,12 +282,4 @@ To run tests
 * Awesome ResponseData class from https://github.com/mikedemers/rbing
 * Work done while working on a project for the company http://reInteractive.net in sunny Sydney.  A great ruby shop should you need help with something.
 
-## TODO
-* pretty light on the tests
-
-## Sample results
-
-See spec/fixtures/*.json for examples of data returned
-
-
 Copyright (c) 2012 Ben Wiseley, released under the MIT license

data/lib/google_custom_search_api.rb

@@ -8,38 +8,83 @@ require 'addressable/uri'
 
 module GoogleCustomSearchApi
   extend self
-  
+
   ##
   # Search the site.
   #
-  # opts 
-  #   see list here for valid options http://code.google.com/apis/customsearch/v1/using_rest.html#query-params
+  # opts
+  #   see list here for valid options
+  #   http://code.google.com/apis/customsearch/v1/using_rest.html#query-params
   def search(query, opts = {})
+    opts[:start] ||= 1
+
+    if (page = opts.delete(:page))
+      page = page.to_i
+      opts[:start] = (page - 1).abs * 10 + 1
+    else
+      page = (opts[:start].to_i / 10) + 1
+    end
+    page = 10 if page > 10
+
     # Get and parse results.
     url = url(query, opts)
-    # puts url
     return nil unless results = fetch(url)
-    results["items"] ||= []
-
-    if file_path =  opts[:save_json_to_file_path]
-      opts[:start] ||= 1
-      Dir.mkdir(file_path) unless Dir.exists?(file_path)
-      fname = "google_#{query.gsub(/[^0-9A-Za-z]/, '_')}_#{opts[:start]}.json"
-      file = File.join(file_path, fname)
-      File.delete(file) if File.exist?(file)
-      open(file,'w') do |f|; f.puts results.to_json; end    
+
+    results['items'] ||= []
+
+    # paging
+    if results.keys.include?('queries')
+      data = results['queries']['request'].first
+      results['pages'] = data['totalResults'].to_i / 10
+      results['pages'] = 10 if results['pages'] > 10
+      results['current_page'] = page.to_i
+
+      results['next_page'] = nil
+      results['previous_page'] = nil
+
+      if results['queries'].include?('nextPage') && page < 10
+        results['next_page'] = results['current_page'] + 1
+      end
+
+      if page > 1
+        results['previous_page'] = page - 1
+      end
     end
 
     ResponseData.new(results)
   end
-  
+
+  ##
+  # Search the site for all available results (max 100)
+  #
+  # This isn't so useful because it's quite slow
+  #
+  # Returns an array of up to 10 search(query) results
+  #
+  # examples:
+  #
+  # results = search_and_return_all_results('poker')
+  # results.first.items.size # == 10
+  #
+  # search_and_resturn_all_results('poker') do |results|
+  #   results.items.size # == 10  10 times
+  # end
+  #
+  # search_and_return_all_results(
+  #   '"California cult winery known for its Rhône"') do |results|
+  #   results.items.size # == 3  1 time
+  # end
+  #
+  # opts
+  #   see list here for valid options
+  #   http://code.google.com/apis/customsearch/v1/using_rest.html#query-params
   def search_and_return_all_results(query, opts = {})
     res = []
     opts[:start] ||= 1
     begin
-      results = GoogleCustomSearchApi.search(query,opts)
-      # results = ResponseData.new(read_search_data("google_poker_#{opts[:start]}"))
-      yield results
+      results = GoogleCustomSearchApi.search(query, opts)
+      return res unless results.keys.include?('queries')
+      yield results if block_given?
       res << results
       if results.queries.keys.include?("nextPage")
         opts[:start] = results.queries.nextPage.first.startIndex
@@ -49,11 +94,7 @@ module GoogleCustomSearchApi
     end while opts[:start].nil? == false
     return res
   end
-  
-  # def read_search_data(fname)
-  #   JSON.parse(File.read("/Users/wiseleyb/dev/rubyx/icm/spec/fixtures/searches/#{fname}.json"))
-  # end
-  
+
   # Convenience wrapper for the response Hash.
   # Converts keys to Strings. Crawls through all
   # member data and converts any other Hashes it
@@ -62,8 +103,11 @@ module GoogleCustomSearchApi
   # to camel case.
   #
   # Usage:
-  # 
-  #  rd = ResponseData.new("AlphaBeta" => 1, "Results" => {"Gamma" => 2, "delta" => [3, 4]})
+  #
+  #  rd = ResponseData.new("AlphaBeta" => 1,
+  #                        "Results" => {
+  #                          "Gamma" => 2,
+  #                          "delta" => [3, 4]})
   #  puts rd.alpha_beta
   #  => 1
   #  puts rd.alpha_beta.results.gamma
@@ -76,6 +120,7 @@ module GoogleCustomSearchApi
     def initialize(data={})
       data.each_pair {|k,v| self[k.to_s] = deep_parse(v) }
     end
+
     def deep_parse(data)
       case data
       when Hash
@@ -86,10 +131,15 @@ module GoogleCustomSearchApi
         data
       end
     end
+
     def method_missing(*args)
       name = args[0].to_s
       return self[name] if has_key? name
-      camelname = name.split('_').map {|w| "#{w[0,1].upcase}#{w[1..-1]}" }.join("")
+
+      camelname = name.split('_').map do |w|
+        "#{w[0,1].upcase}#{w[1..-1]}"
+      end.join("")
+
       if has_key? camelname
         self[camelname]
       else
@@ -97,14 +147,15 @@ module GoogleCustomSearchApi
       end
     end
   end
-  
-  
-  private # -------------------------------------------------------------------
-  
+
+
+  private
+
   ##
   # Build search request URL.
   #
-  # see list here for valid options http://code.google.com/apis/customsearch/v1/using_rest.html#query-params
+  # see list here for valid options
+  # http://code.google.com/apis/customsearch/v1/using_rest.html#query-params
   def url(query, opts = {})
     opts[:q] = query
     opts[:alt] ||= "json"
@@ -114,14 +165,14 @@ module GoogleCustomSearchApi
       params.merge!(GOOGLE_SEARCH_PARAMS)
     rescue NameError
     end
-    "https://www.googleapis.com/customsearch/v1?key=#{GOOGLE_API_KEY}&cx=#{GOOGLE_SEARCH_CX}&#{uri.query}"
+    "https://www.googleapis.com/customsearch/v1?" \
+      "key=#{GOOGLE_API_KEY}&cx=#{GOOGLE_SEARCH_CX}&#{uri.query}"
   end
-  
+
   ##
   # Query Google, and make sure it responds.
   #
   def fetch(url)
     return HTTParty.get(url)
   end
-  
 end

data/lib/version.rb

@@ -1,3 +1,3 @@
 module GoogleCustomSearchApi
-  VERSION = "1.0.0"
+  VERSION = "2.0.0"
 end

metadata

@@ -1,143 +1,136 @@
---- !ruby/object:Gem::Specification 
+--- !ruby/object:Gem::Specification
 name: google_custom_search_api
-version: !ruby/object:Gem::Version 
-  hash: 23
-  prerelease: 
-  segments: 
-  - 1
-  - 0
-  - 0
-  version: 1.0.0
+version: !ruby/object:Gem::Version
+  version: 2.0.0
 platform: ruby
-authors: 
+authors:
 - Ben Wiseley
 autorequire: 
 bindir: bin
 cert_chain: []
-
-date: 2012-08-03 00:00:00 Z
-dependencies: 
-- !ruby/object:Gem::Dependency 
+date: 2016-01-09 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
   name: httparty
-  prerelease: false
-  requirement: &id001 !ruby/object:Gem::Requirement 
-    none: false
-    requirements: 
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
     - - ">="
-      - !ruby/object:Gem::Version 
-        hash: 3
-        segments: 
-        - 0
-        version: "0"
+      - !ruby/object:Gem::Version
+        version: '0'
   type: :runtime
-  version_requirements: *id001
-- !ruby/object:Gem::Dependency 
-  name: addressable
   prerelease: false
-  requirement: &id002 !ruby/object:Gem::Requirement 
-    none: false
-    requirements: 
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: addressable
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
     - - ">="
-      - !ruby/object:Gem::Version 
-        hash: 3
-        segments: 
-        - 0
-        version: "0"
+      - !ruby/object:Gem::Version
+        version: '0'
   type: :runtime
-  version_requirements: *id002
-- !ruby/object:Gem::Dependency 
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
   name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
   prerelease: false
-  requirement: &id003 !ruby/object:Gem::Requirement 
-    none: false
-    requirements: 
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: vcr
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
     - - ">="
-      - !ruby/object:Gem::Version 
-        hash: 3
-        segments: 
-        - 0
-        version: "0"
+      - !ruby/object:Gem::Version
+        version: '0'
   type: :development
-  version_requirements: *id003
-- !ruby/object:Gem::Dependency 
-  name: webmock
   prerelease: false
-  requirement: &id004 !ruby/object:Gem::Requirement 
-    none: false
-    requirements: 
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
     - - ">="
-      - !ruby/object:Gem::Version 
-        hash: 3
-        segments: 
-        - 0
-        version: "0"
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: webmock
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
   type: :development
-  version_requirements: *id004
-- !ruby/object:Gem::Dependency 
-  name: json
   prerelease: false
-  requirement: &id005 !ruby/object:Gem::Requirement 
-    none: false
-    requirements: 
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
     - - ">="
-      - !ruby/object:Gem::Version 
-        hash: 3
-        segments: 
-        - 0
-        version: "0"
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: json
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
   type: :development
-  version_requirements: *id005
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 description: Ruby lib for Google's Custom Search Api.
-email: 
+email:
 - wiseleyb@gmail.com
 executables: []
-
 extensions: []
-
 extra_rdoc_files: []
-
-files: 
+files:
 - CHANGELOG.rdoc
 - Gemfile
 - Gemfile.lock
-- init.rb
 - LICENSE
-- Rakefile
 - README.md
+- Rakefile
+- init.rb
 - lib/google_custom_search_api.rb
 - lib/version.rb
 homepage: http://github.com/wiseleyb/google_custom_search_api
 licenses: []
-
+metadata: {}
 post_install_message: 
 rdoc_options: []
-
-require_paths: 
+require_paths:
 - lib
-required_ruby_version: !ruby/object:Gem::Requirement 
-  none: false
-  requirements: 
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
   - - ">="
-    - !ruby/object:Gem::Version 
-      hash: 3
-      segments: 
-      - 0
-      version: "0"
-required_rubygems_version: !ruby/object:Gem::Requirement 
-  none: false
-  requirements: 
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
   - - ">="
-    - !ruby/object:Gem::Version 
-      hash: 3
-      segments: 
-      - 0
-      version: "0"
+    - !ruby/object:Gem::Version
+      version: '0'
 requirements: []
-
 rubyforge_project: 
-rubygems_version: 1.8.17
+rubygems_version: 2.4.5.1
 signing_key: 
-specification_version: 3
+specification_version: 4
 summary: Ruby lib for Google's Custom Search Api.
 test_files: []
-