google_scraper 0.0.1

data/.gitignore

@@ -0,0 +1,4 @@
+.DS_Store
+results.html
+pkg
+html

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in google_scraper.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2012 John Tajima
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,30 @@
+# GoogleScraper
+
+Scrapes first 10 pages of Google.ca search results and saves url,text into output.csv file.
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+    gem 'google_scraper'
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install google_scraper
+
+## Usage
+
+    $ google_scraper "search term site:somesite.com"
+
+
+## Contributing
+
+1. Fork it
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create new Pull Request

data/README.rdoc

@@ -0,0 +1,19 @@
+= google_scraper - DESCRIBE YOUR GEM
+
+Author::  YOUR NAME (YOUR EMAIL)
+Copyright:: Copyright (c) 2012 YOUR NAME
+
+
+DESCRIBE YOUR GEM HERE
+
+== Links
+
+* {Source on Github}[LINK TO GITHUB]
+* RDoc[LINK TO RDOC.INFO]
+
+== Install
+
+== Examples
+
+== Contributing
+

data/Rakefile

@@ -0,0 +1,61 @@
+def dump_load_path
+  puts $LOAD_PATH.join("\n")
+  found = nil
+  $LOAD_PATH.each do |path|
+    if File.exists?(File.join(path,"rspec"))
+      puts "Found rspec in #{path}"
+      if File.exists?(File.join(path,"rspec","core"))
+        puts "Found core"
+        if File.exists?(File.join(path,"rspec","core","rake_task"))
+          puts "Found rake_task"
+          found = path
+        else
+          puts "!! no rake_task"
+        end
+      else
+        puts "!!! no core"
+      end
+    end
+  end
+  if found.nil?
+    puts "Didn't find rspec/core/rake_task anywhere"
+  else
+    puts "Found in #{path}"
+  end
+end
+require 'bundler'
+require 'rake/clean'
+
+require 'rake/testtask'
+
+require 'cucumber'
+require 'cucumber/rake/task'
+gem 'rdoc' # we need the installed RDoc gem, not the system one
+require 'rdoc/task'
+
+include Rake::DSL
+
+Bundler::GemHelper.install_tasks
+
+
+Rake::TestTask.new do |t|
+  t.pattern = 'test/tc_*.rb'
+end
+
+
+CUKE_RESULTS = 'results.html'
+CLEAN << CUKE_RESULTS
+Cucumber::Rake::Task.new(:features) do |t|
+  t.cucumber_opts = "features --format html -o #{CUKE_RESULTS} --format pretty --no-source -x"
+  t.fork = false
+end
+
+Rake::RDocTask.new do |rd|
+  
+  rd.main = "README.rdoc"
+  
+  rd.rdoc_files.include("README.rdoc","lib/**/*.rb","bin/**/*")
+end
+
+task :default => [:test,:features]
+

data/bin/google_scraper

@@ -0,0 +1,44 @@
+#!/usr/bin/env ruby
+
+require 'optparse'
+require 'methadone'
+require 'google_scraper'
+require 'nokogiri'
+require 'open-uri'
+
+class App
+  include Methadone::Main
+  include Methadone::CLILogging
+
+  main do |query| # Add args you want: |like,so|
+
+    GoogleScraper.run(query)
+  end
+
+  # supplemental methods here
+
+  # Declare command-line interface here
+
+  # description "one line description of your app"
+  #
+  # Accept flags via:
+  # on("--flag VAL","Some flag")
+  # options[flag] will contain VAL
+  #
+  # Specify switches via:
+  # on("--[no-]switch","Some switch")
+  #
+  # Or, just call OptionParser methods on opts
+  #
+  # Require an argument
+  arg :query, :required
+  #
+  # # Make an argument optional
+  # arg :optional_arg, :optional
+
+  version GoogleScraper::VERSION
+
+  use_log_level_option
+
+  go!
+end

data/features/google_scraper.feature

@@ -0,0 +1,13 @@
+Feature: My bootstrapped app kinda works
+  In order to get going on coding my awesome app
+  I want to have aruba and cucumber setup
+  So I don't have to do it myself
+
+  Scenario: App just runs
+    When I get help for "google_scraper"
+    Then the exit status should be 0
+    And the banner should be present
+    And the banner should document that this app takes options
+    And the following options should be documented:
+      |--version|
+    And the banner should document that this app takes no arguments

data/features/step_definitions/google_scraper_steps.rb

@@ -0,0 +1 @@
+# Put your step definitions here

data/features/support/env.rb

@@ -0,0 +1,16 @@
+require 'aruba/cucumber'
+require 'methadone/cucumber'
+
+ENV['PATH'] = "#{File.expand_path(File.dirname(__FILE__) + '/../../bin')}#{File::PATH_SEPARATOR}#{ENV['PATH']}"
+LIB_DIR = File.join(File.expand_path(File.dirname(__FILE__)),'..','..','lib')
+
+Before do
+  # Using "announce" causes massive warnings on 1.9.2
+  @puts = true
+  @original_rubylib = ENV['RUBYLIB']
+  ENV['RUBYLIB'] = LIB_DIR + File::PATH_SEPARATOR + ENV['RUBYLIB'].to_s
+end
+
+After do
+  ENV['RUBYLIB'] = @original_rubylib
+end

data/google_scraper.gemspec

@@ -0,0 +1,25 @@
+# -*- encoding: utf-8 -*-
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'google_scraper/version'
+
+Gem::Specification.new do |gem|
+  gem.name          = "google_scraper"
+  gem.version       = GoogleScraper::VERSION
+  gem.authors       = ["John Tajima"]
+  gem.email         = ["johntajima@gmail.com"]
+  gem.description   = %q{Scrapes first 10 pages of google with given query}
+  gem.summary       = %q{Scrapes first 10 pages of google with given query}
+  gem.homepage      = ""
+
+  gem.files         = `git ls-files`.split($/)
+  gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
+  gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
+  gem.require_paths = ["lib"]
+  gem.add_development_dependency('rdoc')
+  gem.add_development_dependency('aruba')
+  gem.add_development_dependency('rake', '~> 0.9.2')
+  gem.add_dependency('methadone', '~> 1.2.2')
+  gem.add_dependency('nokogiri')
+  
+end

data/lib/google_scraper.rb

@@ -0,0 +1,43 @@
+require "google_scraper/version"
+require 'nokogiri'   
+require 'csv'
+
+module GoogleScraper
+
+  def self.run(query, options = {})
+    puts "Query is #{query}"
+    query = URI.escape(query)
+    data = []
+
+    (0..9).each do |count|
+      puts "scraping page #{count+1} with query #{query}"
+      doc = Nokogiri::HTML(open("https://www.google.ca/search?q=#{query}&start=#{count*10}"))
+
+      link = doc.search('h3')
+      puts "Found #{link.size} possible results"
+
+      link.each do |l| 
+        a = l.search('a').first
+        next unless a['href'].match(/^\/url/)
+        result = {
+          :url => a['href'].gsub(/^\/url\?q\=/,'').gsub(/\&sa=.*$/, ''),
+          :text => a.content
+        }
+        data << result
+      end
+    end
+
+    CSV.open('output.csv', 'w') do |csv|
+      csv << ["url", "text"]
+      data.each do |entry|
+        begin
+          csv << [entry[:url], entry[:text]]
+        rescue => e
+          puts "ugh. #{e} - skipping"
+
+        end
+      end
+    end   
+  end
+
+end

data/lib/google_scraper/version.rb

@@ -0,0 +1,3 @@
+module GoogleScraper
+  VERSION = "0.0.1"
+end

data/output.csv

@@ -0,0 +1,56 @@
+url,text
+http://www.shopify.com/blog/6700752-9-elements-you-need-to-conversion-test,What is Conversion Testing? 9 Website Elements to A/B Test.
+http://support.shopify.com/customer/portal/articles/75397-how-can-i-test-orders-without-being-charged-by-shopify-,Shopify Support: How can I test orders without being char...
+http://wiki.shopify.com/Troubleshooting_PayPal_Website_Payments_Pro,PayPal - Shopify Wiki
+http://ecommerce.shopify.com/c/shopify-discussion/t/testing-migration-from-magento-54355,testing migration from magento - Ecommerce Forums - Shopify
+http://ecommerce.shopify.com/c/ecommerce-marketing/t/a-b-testing-55230,A/B Testing - Ecommerce Forums - Shopify
+http://ecommerce.shopify.com/c/ecommerce-marketing/t/a-b-testing-44736,A/B Testing - Ecommerce Forums - Shopify
+http://ecommerce.shopify.com/c/shopify-discussion/t/testing-105919,Testing - Shopify Discussion - Shopify Ecommerce Discussion Forums
+http://ecommerce.shopify.com/ecommerce/bogus,bogus gateway - Ecommerce Forums - Shopify
+http://static.shopify.com/s/files/1/0079/8612/t/1/assets/PS_Stainless_Steel_Cleaner_8oz.pdf,Stainless Steel - Shopify
+http://static.shopify.com/s/files/1/0079/8612/t/1/assets/PS_Stainless_Steel_Cleaner.pdf,Stainless Steel - Shopify
+http://static.shopify.com/s/files/1/0079/8612/t/1/assets/PS_Glass_Cleaner.pdf,nuveraproducts.com - Shopify
+http://cdn.shopify.com/s/files/1/0088/4082/t/3/assets/Ludaire_Engineered_Flooring_Installation.pdf%3F1644,Engineered Flooring Installation Instructions
+http://ecommerce.shopify.com/c/shopify-discussion/t/smart-collections-acting-dumb-30558,Smart collections acting dumb? - Ecommerce Forums - Shopify
+http://cdn.shopify.com/s/files/1/0017/7152/files/MCN-GB-review-sm.pdf%3F1257882558,Next .. -
+http://cdn.shopify.com/s/files/1/0027/5622/files/Larry_McConkey_comments.pdf%3F1287650375,"June 15, 2005 Excerpt from Steadicam Forum posting I did not want ..."
+http://www.shopify.com/technology/4906352-salmagundi-for-thursday-december-15-2011,"Salmagundi for Thursday, December 15, 2011 — Ecommerce Blog ..."
+http://cdn.shopify.com/s/files/1/0153/9461/files/WHO_IPA_hand_sanitiser_formulations.pdf%3F2206,WHO-recommended Handrub Formulations
+http://static.shopify.com/s/files/1/0077/9972/t/1/assets/Ausblu_Breez_A4.pdf,How do Breez Air Purif iers work? - Shopify
+http://ecommerce.shopify.com/c/ecommerce-design/t/back-button-woes-35677,Back Button Woes - Ecommerce Forums - Shopify
+http://ecommerce.shopify.com/c/ecommerce-design/t/turn-contents-of-an-article-into-a-liquid-array-35152,turn contents of an article into a liquid array..? - Shopify Design ...
+http://static0.shopify.com/s/files/1/0011/2372/files/Toxicity_Testing_FDA-registered_lab_9_06_2007_R1.pdf,Neuon - Shopify
+http://cdn.shopify.com/s/files/1/0061/6332/files/IMDG_Code_35_Changes.pdf%3F1291827878,E
+http://ecommerce.shopify.com/c/ecommerce-design/t/theme-change-9199,Theme Change - Ecommerce Forums - Shopify
+http://cdn.shopify.com/s/files/1/0038/9582/files/ATSAM2195UserGuide.pdf%3F1,ATSAM2195 User Guide.pdf
+http://ecommerce.shopify.com/c/shopify-discussion/t/webhooks-17070,Webhooks - Ecommerce Forums - Shopify
+http://static3.shopify.com/s/files/1/0028/5492/files/Lead_Test.pdf,Report No. : GR:TX:7410062889 DATE : 27/07/2007 - Shopify
+http://ecommerce.shopify.com/c/ecommerce-gallery/t/what-do-you-hate-about-our-store-34897,What do you HATE about our store?! - Ecommerce Forums - Shopify
+http://cdn.shopify.com/s/files/1/0062/7112/files/SocialDesignVolumeOneDigitalEdition.pdf%3F100303,Social Design in Museums
+http://www.shopify.com/technology%3Fpage%3D8,8 - Shopify
+http://wiki.shopify.com/Asset_url,Asset url - Shopify Wiki
+http://support.shopify.com/customer/portal/articles/75397-how-can-i-test-orders-without-being-charged-by-shopify-,Shopify Support: How can I test orders without being char...
+http://cdn.shopify.com/s/files/1/0062/7112/files/SocialDesignVolumeOneDigitalEdition.pdf%3F100303,Social Design in Museums
+http://www.shopify.com/technology%3Fpage%3D8,8 - Shopify
+http://wiki.shopify.com/Asset_url,Asset url - Shopify Wiki
+http://support.shopify.com/customer/portal/articles/75397-how-can-i-test-orders-without-being-charged-by-shopify-,Shopify Support: How can I test orders without being char...
+http://cdn.shopify.com/s/files/1/0062/7112/files/SocialDesignVolumeOneDigitalEdition.pdf%3F100303,Social Design in Museums
+http://www.shopify.com/technology%3Fpage%3D8,8 - Shopify
+http://wiki.shopify.com/Asset_url,Asset url - Shopify Wiki
+http://support.shopify.com/customer/portal/articles/75397-how-can-i-test-orders-without-being-charged-by-shopify-,Shopify Support: How can I test orders without being char...
+http://cdn.shopify.com/s/files/1/0062/7112/files/SocialDesignVolumeOneDigitalEdition.pdf%3F100303,Social Design in Museums
+http://www.shopify.com/technology%3Fpage%3D8,8 - Shopify
+http://wiki.shopify.com/Asset_url,Asset url - Shopify Wiki
+http://support.shopify.com/customer/portal/articles/75397-how-can-i-test-orders-without-being-charged-by-shopify-,Shopify Support: How can I test orders without being char...
+http://cdn.shopify.com/s/files/1/0062/7112/files/SocialDesignVolumeOneDigitalEdition.pdf%3F100303,Social Design in Museums
+http://www.shopify.com/technology%3Fpage%3D8,8 - Shopify
+http://wiki.shopify.com/Asset_url,Asset url - Shopify Wiki
+http://support.shopify.com/customer/portal/articles/75397-how-can-i-test-orders-without-being-charged-by-shopify-,Shopify Support: How can I test orders without being char...
+http://cdn.shopify.com/s/files/1/0062/7112/files/SocialDesignVolumeOneDigitalEdition.pdf%3F100303,Social Design in Museums
+http://www.shopify.com/technology%3Fpage%3D8,8 - Shopify
+http://wiki.shopify.com/Asset_url,Asset url - Shopify Wiki
+http://support.shopify.com/customer/portal/articles/75397-how-can-i-test-orders-without-being-charged-by-shopify-,Shopify Support: How can I test orders without being char...
+http://cdn.shopify.com/s/files/1/0062/7112/files/SocialDesignVolumeOneDigitalEdition.pdf%3F100303,Social Design in Museums
+http://www.shopify.com/technology%3Fpage%3D8,8 - Shopify
+http://wiki.shopify.com/Asset_url,Asset url - Shopify Wiki
+http://support.shopify.com/customer/portal/articles/75397-how-can-i-test-orders-without-being-charged-by-shopify-,Shopify Support: How can I test orders without being char...

data/test/tc_something.rb

@@ -0,0 +1,7 @@
+require 'test/unit'
+
+class TestSomething < Test::Unit::TestCase
+  def test_truth
+    assert true
+  end
+end

metadata

@@ -0,0 +1,120 @@
+--- !ruby/object:Gem::Specification
+name: google_scraper
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+  prerelease: 
+platform: ruby
+authors:
+- John Tajima
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2012-11-22 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: rdoc
+  requirement: &70194036751580 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: *70194036751580
+- !ruby/object:Gem::Dependency
+  name: aruba
+  requirement: &70194036751140 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: *70194036751140
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: &70194036750600 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.9.2
+  type: :development
+  prerelease: false
+  version_requirements: *70194036750600
+- !ruby/object:Gem::Dependency
+  name: methadone
+  requirement: &70194036750060 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 1.2.2
+  type: :runtime
+  prerelease: false
+  version_requirements: *70194036750060
+- !ruby/object:Gem::Dependency
+  name: nokogiri
+  requirement: &70194036749660 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: *70194036749660
+description: Scrapes first 10 pages of google with given query
+email:
+- johntajima@gmail.com
+executables:
+- google_scraper
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- Gemfile
+- LICENSE.txt
+- README.md
+- README.rdoc
+- Rakefile
+- bin/google_scraper
+- features/google_scraper.feature
+- features/step_definitions/google_scraper_steps.rb
+- features/support/env.rb
+- google_scraper.gemspec
+- lib/google_scraper.rb
+- lib/google_scraper/version.rb
+- output.csv
+- test/tc_something.rb
+homepage: ''
+licenses: []
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.11
+signing_key: 
+specification_version: 3
+summary: Scrapes first 10 pages of google with given query
+test_files:
+- features/google_scraper.feature
+- features/step_definitions/google_scraper_steps.rb
+- features/support/env.rb
+- test/tc_something.rb