generalscraper 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 1e568d1e9d1c0fa9c98f814128439bd009194c0d
+  data.tar.gz: f26d58116e50b61ad308eed4df0ce03e76cbfba8
+SHA512:
+  metadata.gz: 92120914fbe80f8a0b5b3b5b996753b07a218745a88d7cd201fb533fa3de4e116499c1cc2d218d030771bf3d25dbf42743efc83ed7f1d2e4cb44939229ec31a6
+  data.tar.gz: b7d4070620f7a43ff7008ae414ea3aed53da287c697f32cb80f6afbd66186d94151a7393ac9547428f9ddeabf37e3500e7e55165ad11f7d2e1f994975e539f96

data/lib/generalscraper.rb

@@ -0,0 +1,78 @@
+require 'mechanize'
+require 'json'
+require 'nokogiri'
+require 'open-uri'
+
+class GeneralScraper
+  def initialize(scrapesite, input)
+   @input = input
+   @scrapesite = scrapesite
+   @output = Array.new
+   @startindex = 10
+  end
+
+  # Searches for links on Google
+  def search
+    agent = Mechanize.new
+    agent.user_agent_alias = 'Linux Firefox'
+    gform = agent.get("http://google.com").form("f")
+    gform.q = "site:" + @scrapesite + " " + @input
+    page = agent.submit(gform, gform.buttons.first)
+    examine(page)
+  end
+ 
+  # Examines a search page
+  def examine(page)
+    page.links.each do |link|
+      if (link.href.include? @scrapesite) && (!link.href.include? "webcache") && (!link.href.include? "site:"+@scrapesite)
+        saveurl = link.href.split("?q=")
+        
+        if saveurl[1]
+          url = saveurl[1].split("&")
+          getPage(url[0])
+        end
+      end
+
+      if (link.href.include? "&sa=N") && (link.href.include? "&start=")
+        url1 = link.href.split("&start=")
+        url2 = url1[1].split("&sa=N")
+
+        if url2[0].to_i == @startindex
+          sleep(rand(30..90))
+          @startindex += 10
+          agent = Mechanize.new
+          examine(agent.get("http://google.com" + link.href))
+        end
+      end
+    end
+  end
+
+  # Scrape the page content
+  def getPage(url)
+    pagehash = Hash.new
+    begin
+      url.gsub!("%3F", "?")
+      url.gsub!("%3D", "=")
+      pagehash[:url] = url
+      pagehash[:date_retrieved] = Time.now
+      html = Nokogiri::HTML(open(url))
+      pagehash[:title] = html.css("title").text
+      html.css("meta").each do |m|
+        if m
+          pagehash[m['name']] = m['content']
+        end
+      end
+      pagehash[:page] = html.css("body").text
+      @output.push(pagehash)
+    rescue
+      puts "URL: " + url
+    end
+  end
+
+  # Gets all data and returns in JSON
+  def getData
+    search
+    return JSON.pretty_generate(@output)
+  end
+end
+

metadata

@@ -0,0 +1,45 @@
+--- !ruby/object:Gem::Specification
+name: generalscraper
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- M. C. McGrath
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2014-05-15 00:00:00.000000000 Z
+dependencies: []
+description: Scrapes all pages on a site you specify including terms you specify.
+email: shidash@shidash.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- lib/generalscraper.rb
+homepage: https://github.com/TransparencyToolkit/generalscraper
+licenses:
+- GPL
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.0.14
+signing_key: 
+specification_version: 4
+summary: Get all pages on a site for terms specified
+test_files: []
+has_rdoc: