google-site-search 0.0.2

data/.gitignore

@@ -0,0 +1,17 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp

data/Gemfile

@@ -0,0 +1,9 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in google-site-search.gemspec
+gemspec
+
+group :test do
+  gem "guard-minitest"
+  gem "rb-inotify"
+end

data/Guardfile

@@ -0,0 +1,24 @@
+# A sample Guardfile
+# More info at https://github.com/guard/guard#readme
+
+guard 'minitest', :notify => false do
+  # with Minitest::Unit
+  watch(%r|^test/(.*)\/?test_(.*)\.rb|)
+  watch(%r|^lib/(.*)([^/]+)\.rb|)     { |m| "test/#{m[1]}test_#{m[2]}.rb" }
+  watch(%r|^test/test_helper\.rb|)    { "test" }
+
+  # with Minitest::Spec
+  # watch(%r|^spec/(.*)_spec\.rb|)
+  # watch(%r|^lib/(.*)([^/]+)\.rb|)     { |m| "spec/#{m[1]}#{m[2]}_spec.rb" }
+  # watch(%r|^spec/spec_helper\.rb|)    { "spec" }
+
+  # Rails 3.2
+  # watch(%r|^app/controllers/(.*)\.rb|) { |m| "test/controllers/#{m[1]}_test.rb" }
+  # watch(%r|^app/helpers/(.*)\.rb|)     { |m| "test/helpers/#{m[1]}_test.rb" }
+  # watch(%r|^app/models/(.*)\.rb|)      { |m| "test/unit/#{m[1]}_test.rb" }  
+  
+  # Rails
+  # watch(%r|^app/controllers/(.*)\.rb|) { |m| "test/functional/#{m[1]}_test.rb" }
+  # watch(%r|^app/helpers/(.*)\.rb|)     { |m| "test/helpers/#{m[1]}_test.rb" }
+  # watch(%r|^app/models/(.*)\.rb|)      { |m| "test/unit/#{m[1]}_test.rb" }  
+end

data/LICENSE

@@ -0,0 +1,22 @@
+Copyright (c) 2012 David Vallance
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.rdoc

@@ -0,0 +1,145 @@
+= Gem: google-site-search
+
+== Description
+
+This gem was created to aid in the querying and parsing of the {Google Site Search}[http://www.google.com/sitesearch] api. 
+
+In the simplest use case it will query your google site search for a term and supply you with an object containing the results. However I've built this gem with the intention that you will want to explicitly handle _what_ and _how_ the specific results are stored. 
+
+== Installation
+
+Add the following to your projects Gemfile.
+
+	gem 'google-site-search', :git => "git@github.com:dvallance/google-site-search.git"
+
+Require the code if necessary (_note:_ some frameworks like rails are set to auto-require gems for you by default)
+
+	require 'google-site-search'
+
+== Usage
+
+The simpliest way to use the gem is by providing just a *search* *query* *term* and your *search* *engine* *unique* *id* code (_e.g._ looks like this +00255077836266642015+:+u-scht7a-8i+ and is located in your google site search control panel) 
+
+	#just assign the query to an object
+	search = GoogleSiteSearch.query(GoogleSiteSearch::UrlBuilder.new("microsoft", "00255077836266642015:u-scht7a-8i")
+  
+	#object has search attributes like
+	puts search.next_results_url
+	puts search.previous_results_url
+	puts search.xml
+	puts search.spelling
+	puts search.spelling_url
+
+	#object has an array of each specific result that contains title, description and its link by default
+	search.results.each do |result|
+		puts result.title
+		puts result.description
+		puts result.link
+	end
+	
+The _query_ method expects a valid url so if you wanted to supply your own you can! However I have created a builder class to help with proper url creation and to help do some of the work for you.
+
+== Advanced Usage 
+
+An important requirement for this gem was to be able to use {structured data}[https://developers.google.com/custom-search/docs/structured_data] for:
+* querying the search api itself (_i.e._ filtering and sorting )
+* displaying specific information in views (_i.e._ display a specific field like'author', or 'product_type')
+
+Therefore I allow the developer to supply his own "*Results*" class to the query and allow them to parse each result xml element explicitly.
+
+The default Result class is as follows:
+
+	class Result
+		attr_reader :title, :link, :description
+
+		def initialize(node)
+			@title = node.find_first("T").content
+			@link = node.find_first("UE").content
+			@description = node.find_first("S").content
+		end
+	end
+
+As you can see it is very simple. Your class simply needs an initialize method that will recieve an xml node, which it can then do with as it pleases. After it is initialized it is added to the _search.results_ array as shown previously.
+
+See
+* {libxml-ruby}[http://libxml.rubyforge.org/rdoc/] for help on xml parsing.
+* {Googles Site Search XML API reference}[https://developers.google.com/custom-search/docs/xml_results?hl=en] for the <R> tag which encapsulates the details of an individual search result.
+
+== Pagination
+
+The google search api actually does the work of pagination for us, supplying the next and previous urls. The urls are relative to \http://www.google.com so I added a _paginate_ method to simplify the call.
+
+	search2 = GoogleSiteSearch.query(GoogleSiteSearch.paginate(search1.next_results_url))
+
+== Pagination Simple Example
+
+This works and is fairly straight forward.
+
+In your controller:
+
+    if params[:move]
+      @search = GoogleSiteSearch.query(GoogleSiteSearch.paginate(params[:move]))
+    else
+      @search = GoogleSiteSearch.query(GoogleSiteSearch::UrlBuilder.new("microsoft", "00255077836266642015:u-scht7a-8i", :num => 5))
+    end
+
+In your view:
+
+	<% if @search.previous_results_url %>
+		<%= link_to "Previous", search_url(:move => @search.previous_results_url) %>
+	<% end %>
+	<% if @search.next_results_url %>
+		<%= link_to "More", search_url(:move => @search.next_results_url) %>
+	<% end %>
+
+== Escaping
+
+If you start passing around the url's in parameters you may run into issues if you don't escape/unescape the url. If so try...
+
+View adds escape:
+
+	<%= link_to "Previous", search_url(:move => CGI::escape(@search.previous_results_url)) %>
+
+Controller unescapes:
+
+	@search = GoogleSiteSearch.query(GoogleSiteSearch.paginate(CGI::unescape(params[:move])))
+
+
+
+== Filtering and Sorting
+
+See {Filtering and sorting search results.}[https://developers.google.com/custom-search/docs/structured_search]
+
+=== Filtering
+
+Google expects filtering to be on the "search query" itself. However I feel my end users won't and shouldn't be aware of all the possible filtering options (most of my filtering will be based off of dataobject values I supply myself). So I try and keep the filters and actual "search term" separate. 
+
+From the google reference link above an example filter search query is <b>halloween more:pagemap:document-author:lisamorton</b>
+
+	#using the example above would look like this.
+	search = GoogleSiteSearch.query(GoogleSiteSearch::UrlBuilder.new("halloween", "00255077836266642015:u-scht7a-8i", :filter =>  "more:pagemap:document-author:lisamorton")
+
+=== Separate Search Term From Filters
+
+The full "search query" is returned by google's api and stored in the Search object in a few spots. (i.e <i>@search.search_query</i> method and <i>@search.spelling_q</i>).
+
+To separate the search term from the filter use:
+
+	search_term, filters = GoogleSiteSearch.separate_search_term_from_filters(@search.search_query)
+
+=== Sorting
+
+Sorting would also be done by specifing a *sort* option. 
+
+	search = GoogleSiteSearch.query(GoogleSiteSearch::UrlBuilder.new("halloween", "00255077836266642015:u-scht7a-8i", :filter =>  "more:pagemap:document-author:lisamorton", :sort => "data-sdate")
+	
+== Other Params
+
+Any <b>[param=value]</b> query string additions you want to add can be assigned like the sorting above. For example to limit the search results return, to 5, would look like...
+
+	#get only 5 search results with the filtering and sorting from above still applyed.
+	search = GoogleSiteSearch.query(GoogleSiteSearch::UrlBuilder.new("halloween more:pagemap:document-author:lisamorton", "00255077836266642015:u-scht7a-8i", :sort => "date-sdate", :num => "5" )
+
+== Author
+
+David Vallance

data/Rakefile

@@ -0,0 +1,2 @@
+#!/usr/bin/env rake
+require "bundler/gem_tasks"

data/google-site-search.gemspec

@@ -0,0 +1,19 @@
+# -*- encoding: utf-8 -*-
+require File.expand_path('../lib/google-site-search/version', __FILE__)
+
+Gem::Specification.new do |gem|
+  gem.authors       = ["David Vallance"]
+  gem.email         = ["davevallance@gmail.com"]
+  gem.description   = %q{A gem to aid in the consumption of the google site search service; querys the service, populates a result object and has some related helper methods.}
+  gem.summary       = gem.description 
+  gem.homepage      = "https://github.com/dvallance/google-site-search"
+
+  gem.files         = `git ls-files`.split($\)
+  gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
+  gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
+  gem.name          = "google-site-search"
+  gem.require_paths = ["lib"]
+  gem.version       = GoogleSiteSearch::VERSION
+  gem.add_dependency("activesupport")
+  gem.add_dependency("libxml-ruby")
+end

data/lib/google-site-search.rb

@@ -0,0 +1,79 @@
+require "active_support/core_ext/object/to_query"
+require "active_support/core_ext/hash/indifferent_access"
+require "active_support/core_ext/object/try"
+require "active_support/core_ext/object/blank"
+require "google-site-search/version"
+require "google-site-search/url_builder"
+require "google-site-search/search"
+require "google-site-search/result"
+require "timeout"
+require "net/http"
+require "uri"
+require "xml"
+
+##
+# A module to help query and parse the google site search api.
+#
+module GoogleSiteSearch
+  
+	
+  GOOGLE_SEARCH_URL = "http://www.google.com"
+  DEFAULT_PARAMS = {
+    :client => "google-csbe",
+    :output => "xml_no_dtd",
+  }
+
+  class << self
+
+    # Expects the URL returned by Search#next_results_url or Search#previous_results_url.
+    def paginate url
+      GOOGLE_SEARCH_URL + url.to_s
+    end
+
+    # See Search - This is a convienence method for creating and querying. 
+    def query url, result_class = Result
+      Search.new(url, result_class).query
+    end
+
+		def query_multiple url, result_class = Result, times
+			searchs = []
+			while times > 0
+				times -= 1
+				url = paginate(searchs.last.try(:next_results_url)) unless searchs.empty?
+				searchs << Search.new(url, result_class).query
+			end
+			searchs
+		end
+
+    # Makes a request to the google search api and returns the xml response as a string.
+    def request_xml url
+			response = nil
+      begin
+        ::Timeout::timeout(2) do
+          response = Net::HTTP.get_response(URI.parse(url.to_s))
+        end
+      rescue Errno::ECONNREFUSED => err
+			rescue URI::InvalidURIError => err
+				puts "URI Error [#{url}]"
+      rescue => err
+     	  puts "ERROR #{err.class}"  
+      end
+			response.body if response && response.code == "200"
+    end
+
+    # Google returns a result link as an absolute but you may
+    # want a relative version. 
+    def relative_path path
+      uri = URI.parse(path)
+      uri.relative? ? path : [uri.path,uri.query].compact.join("?")
+    end
+
+		# Google's api will give back a full query which has the filter options on it. I like to deal with them separately so this method breaks them up.
+		def separate_search_term_from_filters(string)
+			match = /\smore:p.*/.match(string)
+			return [string, nil] if match.nil?
+			return [match.pre_match, match[0]] 
+		end
+
+  end
+end

data/lib/google-site-search/result.rb

@@ -0,0 +1,22 @@
+module GoogleSiteSearch
+
+	# A default class that parses a result element from 
+	# Googles search API.
+	#
+	# See {LibXML Ruby's Node}[http://libxml.rubyforge.org/rdoc/classes/LibXML/XML/Node.html] when writing your own Result class.
+	class Result
+		attr_reader :title, :link, :description
+		
+		# ==== Attributes
+		#
+		# * +node+ - LibXML::XML::Node.
+		def initialize(node)
+			@title = node.find_first("T").content
+
+			# Fully qualified URL to the result.
+			@link = node.find_first("UE").content
+
+			@description = node.find_first("S").content
+		end
+	end
+end

data/lib/google-site-search/search.rb

@@ -0,0 +1,80 @@
+module GoogleSiteSearch
+
+	# Search is responsible for parsing the returned xml from
+	# google's API.
+	#
+	# XML parsing is done using {LibXML Ruby}[http://libxml.rubyforge.org/rdoc/]
+  class Search
+		# Goolge Site Search API url.
+    attr_reader :url
+		# Array of *result_class* objects.
+    attr_reader :results
+		# Spelling suggestion in HTML format.
+    attr_reader :spelling
+		# Spelling suggestion URL escaped.
+    attr_reader :spelling_q
+		# Pulled from the XML as the estimated total number of results.
+		# *Note* Google themselves say this may not be accurate. 
+    attr_reader :estimated_results_total
+		# Relative URL to get the next set of results (if any).
+    attr_reader :next_results_url
+		# Relative URL to get the previous set of results (if any).
+    attr_reader :previous_results_url
+		# String of the xml returned by Google.
+    attr_reader :xml
+		# Class supplied which is responsible for parsing each
+		# individual result from the API XML.
+    attr_reader :result_class
+		# The full search term + filters query google interpreted from
+		# the url supplied.
+    attr_reader :search_query
+
+
+		# ==== Attributes
+		#
+		# * +url+ - Expects a fully qualified url to Googles search API
+		#   (can be a string or from an objects to_s method).
+		# * +reulst_class+ - A class that's initialize method is expected
+		#   to handle the parsing of an individual result entry. 
+    def initialize url, result_class
+      @url = url.to_s
+      @results = Array.new
+      @result_class = result_class
+    end
+
+		# Query's Google API, stores the xml and parses values into itself.
+    def query
+      @xml = GoogleSiteSearch::request_xml(url)
+      parse_xml unless @xml.nil?
+			self
+    end
+
+    def results
+      @results || []
+    end
+
+
+    private
+
+    def parse_xml
+      begin
+        doc = ::XML::Parser.string(@xml).parse
+        doc.find("//GSP/RES/R").each do |result_node|
+          @results << result_class.new(result_node)
+          #puts "*** attribute =  #{result_node.find_first("PageMap/DataObject[@type='metatags']/Attribute[@name='title']").attributes[:value] }"
+        end
+
+        spelling_node = doc.find_first("Spelling/Suggestion")
+				@spelling = spelling_node.try(:content) 
+				@spelling_q = spelling_node.try(:attributes).try(:[],:q)
+        @estimated_results_total = doc.find_first("RES/M").try(:content)
+        @next_results_url = doc.find_first("RES/NB/NU").try(:content)
+        @previous_results_url = doc.find_first("RES/NB/PU").try(:content)
+        @search_query = doc.find_first("Q").try(:content)
+
+      rescue => err
+         puts "parse_xml error #{err.message}"
+      end
+    end
+  end
+end

data/lib/google-site-search/url_builder.rb

@@ -0,0 +1,42 @@
+module GoogleSiteSearch
+
+	# UrlBuilder is responsible for generating a valid url for querying
+	# the google search API.
+  class UrlBuilder
+		attr_accessor :search_term, :filter, :custom_params, :search_engine_id, :sort
+
+		# ==== Attributes
+		#
+		# * +search_term+ - should be a simple search with *no* *filters* added.
+		#
+		# Even though Googles search API uses filters on the search term
+		# itself, I wanted to separate out that functionality.
+		# Filters should be added separately as a param.
+		#
+		# * +search_engine_id+ - your unique id found in your Google Site Search
+		#   control panel.
+		#
+		# * +params+ - supply a hash that is converted to query params. See
+		#   {Request Params}[https://developers.google.com/custom-search/docs/xml_results?hl=en#wsRequestParameters].
+    def initialize search_term, search_engine_id, params = {}
+			raise ArgumentError if search_term.blank? || search_engine_id.blank?
+      @search_term = search_term
+      @search_engine_id = search_engine_id
+      @custom_params = params.with_indifferent_access
+			@filter = @custom_params.delete :filter
+			@sort = @custom_params.delete :sort 
+    end
+
+		# Joins the search term and the filters, to get the full search query that google expects.
+		def query
+			[@search_term,@filter].compact.join(" ")
+		end
+		
+		# Returns a fully qualified URL for the Google search API.
+    def url
+      "#{GOOGLE_SEARCH_URL}/cse?#{@custom_params.merge(DEFAULT_PARAMS).merge(:q => query, :cx => @search_engine_id, :sort => @sort).delete_if{|k,v| v.nil?}.to_query}" 
+    end
+		alias :to_s :url
+
+  end
+end

data/lib/google-site-search/version.rb

@@ -0,0 +1,3 @@
+module GoogleSiteSearch
+	VERSION = "0.0.2"
+end

data/test/test_google_site_search.rb

@@ -0,0 +1,37 @@
+require_relative 'test_helper'
+
+describe GoogleSiteSearch do
+
+  describe '#.relative_path' do
+
+    it 'raises InvalidURLError if a nil is given' do
+      -> {GoogleSiteSearch.relative_path(nil)}.must_raise URI::InvalidURIError
+    end
+
+    describe 'given a relative path' do
+
+      it 'returns the path given' do
+        GoogleSiteSearch.relative_path("/somepath").must_equal "/somepath" 
+      end
+      
+    end 
+
+    describe 'given an absolute url' do 
+
+      it 'with just the domain a root path will be returned' do
+        GoogleSiteSearch.relative_path("http://www.somesite.com/").must_equal "/" 
+      end
+
+      it 'with a domain and path given the path will be returned' do
+        GoogleSiteSearch.relative_path("http://www.somesite.com/my-test").must_equal "/my-test" 
+
+      end
+
+      it 'with a query string the path and query will be returned' do
+        GoogleSiteSearch.relative_path("http://www.somesite.com/my-test?something=value").must_equal "/my-test?something=value" 
+
+      end
+
+    end
+  end
+end

data/test/test_helper.rb

@@ -0,0 +1,5 @@
+ENV['RACK_ENV'] = 'test'
+
+require "google-site-search"
+require "minitest/spec"
+include GoogleSiteSearch #include module so I don't have to namespace everything

data/test/test_search.rb

@@ -0,0 +1,7 @@
+require_relative 'test_helper'
+
+describe Search do
+
+
+
+end

data/test/test_url_builder.rb

@@ -0,0 +1,13 @@
+require_relative 'test_helper'
+
+describe UrlBuilder do
+	it "#initialize expects valid arguments" do
+		-> {GoogleSiteSearch::UrlBuilder.new(nil, nil)}.must_raise ArgumentError
+		-> {GoogleSiteSearch::UrlBuilder.new("","")}.must_raise ArgumentError
+		-> {GoogleSiteSearch::UrlBuilder.new("string", nil)}.must_raise ArgumentError
+		-> {GoogleSiteSearch::UrlBuilder.new(nil, "string")}.must_raise ArgumentError
+		GoogleSiteSearch::UrlBuilder.new("string", "string").must_be_instance_of UrlBuilder
+	end
+
+end
+

metadata

@@ -0,0 +1,89 @@
+--- !ruby/object:Gem::Specification
+name: google-site-search
+version: !ruby/object:Gem::Version
+  version: 0.0.2
+  prerelease: 
+platform: ruby
+authors:
+- David Vallance
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2012-10-25 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: activesupport
+  requirement: &7621180 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: *7621180
+- !ruby/object:Gem::Dependency
+  name: libxml-ruby
+  requirement: &7620760 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: *7620760
+description: A gem to aid in the consumption of the google site search service; querys
+  the service, populates a result object and has some related helper methods.
+email:
+- davevallance@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- Gemfile
+- Guardfile
+- LICENSE
+- README.rdoc
+- Rakefile
+- google-site-search.gemspec
+- lib/google-site-search.rb
+- lib/google-site-search/result.rb
+- lib/google-site-search/search.rb
+- lib/google-site-search/url_builder.rb
+- lib/google-site-search/version.rb
+- test/test_google_site_search.rb
+- test/test_helper.rb
+- test/test_search.rb
+- test/test_url_builder.rb
+homepage: https://github.com/dvallance/google-site-search
+licenses: []
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.10
+signing_key: 
+specification_version: 3
+summary: A gem to aid in the consumption of the google site search service; querys
+  the service, populates a result object and has some related helper methods.
+test_files:
+- test/test_google_site_search.rb
+- test/test_helper.rb
+- test/test_search.rb
+- test/test_url_builder.rb