generalscraper 0.0.3 → 0.0.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 84330f1a3a5c18dd9b2d8b200b141b5ba7c85827
-  data.tar.gz: 5fe4f81df62c28962565340858984e3d6b6f8a86
+  metadata.gz: 48ee021e7ac6bb45a00308d69003bd6ba379b20b
+  data.tar.gz: d3b631127266dbfaacaee4eb74c2868e48a1f0c8
 SHA512:
-  metadata.gz: 58b032cb6a3f33f4e5f1d972147952906ca95be1fbbbf181afad95336cc4a24b7585733305e8eefe9bce38da77cb989510a41020b9d418d93b68c7ef2fc8c1e1
-  data.tar.gz: 401093d88ca984e4a3fdc5f39f3ec35be9441fdff15e23fd434d623203ec22a9687f98d5fcfdd021c90662eb791f8ec34e57d45d59d08b5364b626a1961d6865
+  metadata.gz: b15d9ce46f5223be79fca5ba74423c0eab88c03dc3ed1e40baef500d30ab9f15c1f364bfb23244ea1dc741edcd91281b779b4ff1170341f0c534859aa174ff94
+  data.tar.gz: 149dadfabb77b586164c4213fd58bca33a5de5d0c64af48c04db6f4e47eaf3c5c1563ceaeedd7e9a97c813e7e5b95cc45a671734b8a5d2b78212db0d30d700ed

data/lib/generalscraper.rb

@@ -1,90 +1,71 @@
-require 'mechanize'
 require 'json'
 require 'nokogiri'
-require 'open-uri'
-require 'uploadconvert'
+require 'mechanize'
+
+load 'parse_page.rb'
+load 'proxy_manager.rb'
 
 class GeneralScraper
-  def initialize(scrapesite, input, table)
-   @input = input
-   @scrapesite = scrapesite
-   @output = Array.new
-   @startindex = 10
-   @table = table
+  include ParsePage
+  include ProxyManager
+  
+  def initialize(operators, searchterm, proxylist)
+    @operators = operators
+    @searchterm = searchterm
+    @op_val = @operators.split(" ")[0].split(":")[1]
+    @proxylist = IO.readlines(proxylist)
+    @usedproxies = Hash.new
+    
+    @output = Array.new
+    @urllist = Array.new
+    @startindex = 10
   end
 
   # Searches for links on Google
   def search
-    agent = Mechanize.new
-    agent.user_agent_alias = 'Linux Firefox'
-    gform = agent.get("http://google.com").form("f")
-    gform.q = "site:" + @scrapesite + " " + @input
-    page = agent.submit(gform, gform.buttons.first)
-    examine(page)
+    categorizeLinks(getPage("http://google.com", @operators + " " + @searchterm))
   end
- 
-  # Examines a search page
-  def examine(page)
-    page.links.each do |link|
-      if (link.href.include? @scrapesite) && (!link.href.include? "webcache") && (!link.href.include? "site:"+@scrapesite)
-        saveurl = link.href.split("?q=")
-        
-        if saveurl[1]
-          url = saveurl[1].split("&")
-          getPage(url[0])
-        end
-      end
-
-      if (link.href.include? "&sa=N") && (link.href.include? "&start=")
-        url1 = link.href.split("&start=")
-        url2 = url1[1].split("&sa=N")
 
-        if url2[0].to_i == @startindex
-          sleep(rand(30..90))
-          @startindex += 10
-          agent = Mechanize.new
-          examine(agent.get("http://google.com" + link.href))
-        end
+  # Categorizes the links on results page into results and other search pages
+  def categorizeLinks(page)
+    page.links.each do |link|
+      if (link.href.include? @op_val) && (!link.href.include? "webcache") && (!link.href.include? @operators.gsub(" ", "+"))
+        siteURLSave(link)
+      elsif (link.href.include? "&sa=N") && (link.href.include? "&start=")
+        nextSearchPage(link)
       end
     end
   end
 
-  # Scrape the page content
-  def getPage(url)
-    pagehash = Hash.new
-    begin
-      url.gsub!("%3F", "?")
-      url.gsub!("%3D", "=")
-      pagehash[:url] = url
-      pagehash[:date_retrieved] = Time.now
-      html = Nokogiri::HTML(open(url))
-      pagehash[:title] = html.css("title").text
-      html.css("meta").each do |m|
-        if m
-          pagehash[m['name']] = m['content']
-        end
-      end
-      if @table == false
-        if url.include? ".pdf"
-          `wget -P public/uploads #{url}`
-          path = url.split("/")
-          u = UploadConvert.new("public/uploads/" + path[path.length-1].chomp.strip)
-          pdfparse = JSON.parse(u.handleDoc)
-          pdfparse.each{|k, v| pagehash[k] = v}
-        else
-          pagehash[:text] = html.css("body").text
-        end
-      end
-      @output.push(pagehash)
-    rescue
-      
+  # Parse and save the URLs for search results
+  def siteURLSave(link)
+    site_url = link.href.split("?q=")[1]
+    @urllist.push(site_url.split("&")[0]) if site_url
+  end
+
+  # Process search links and go to next page
+  def nextSearchPage(link)
+    page_index_num = link.href.split("&start=")[1].split("&sa=N")[0]
+    
+    if page_index_num.to_i == @startindex
+      @startindex += 10
+      categorizeLinks(getPage("http://google.com" + link.href + "&filter=0"))
     end
   end
 
   # Gets all data and returns in JSON
   def getData
     search
+    @urllist.each do |url|
+      getPageData(url)
+    end
     return JSON.pretty_generate(@output)
   end
+
+  # Returns a list of search result URLs
+  def getURLs
+    search
+    return JSON.pretty_generate(@urllist)
+  end
 end
 

data/lib/parse_page.rb

@@ -0,0 +1,63 @@
+require 'uploadconvert'
+
+module ParsePage
+  # Get both page metadata and text
+  def getPageData(url)
+    begin
+      pagehash = getMetadata(url)
+      pagehash = getContent(url, pagehash)
+      @output.push(pagehash)
+    rescue
+    end
+  end
+
+  # Get the page content by type of page
+  def getContent(url, pagehash)
+    if url.include? ".pdf"
+      return getPDF(url, pagehash)
+    else
+      return getHTMLText(url, pagehash)
+    end
+  end
+
+  # Download the page text
+  def getHTMLText(url, pagehash)
+    html = Nokogiri::HTML(getPage(url).body)
+    pagehash[:text] = html.css("body").text
+    return pagehash
+  end
+
+  # Download and extract text from PDF
+  def getPDF(url, pagehash)
+    `wget -P public/uploads #{url}`
+    path = url.split("/")
+
+    # OCR PDF and save fields
+    u = UploadConvert.new("public/uploads/" + path[path.length-1].chomp.strip)
+    pdfparse = JSON.parse(u.handleDoc)
+    pdfparse.each{|k, v| pagehash[k] = v}
+    return pagehash
+  end
+
+  # Get the page metadata
+  def getMetadata(url)
+    pagehash = Hash.new
+
+    # Save URL and date retreived
+    url.gsub!("%3F", "?")
+    url.gsub!("%3D", "=")
+    pagehash[:url] = url
+    pagehash[:date_retrieved] = Time.now
+
+    # Get title and meta tag info
+    html = Nokogiri::HTML(getPage(url).body) # Eventually modify this
+    pagehash[:title] = html.css("title").text
+    html.css("meta").each do |m|
+      if m
+        pagehash[m['name']] = m['content']
+      end
+    end
+
+    return pagehash
+  end
+end

data/lib/proxy_manager.rb

@@ -0,0 +1,35 @@
+require 'active_support/time'
+require 'mechanize'
+
+module ProxyManager 
+  # Get the page with a proxy
+  def getPage(url, form_input = nil)
+    agent = Mechanize.new do |a|
+      a.user_agent_alias = "Linux Firefox"
+      a.set_proxy(getRandomProxy, 80)
+    end
+
+    if form_input
+      gform = agent.get(url).form("f")
+      gform.q = form_input
+      return agent.submit(gform, gform.buttons.first)
+    else
+      return agent.get(url)
+    end
+  end
+
+  # Choose a random proxy
+  def getRandomProxy
+    max = @proxylist.length
+    chosen = @proxylist[Random.rand(max)]
+
+    # Only use proxy if it hasn't been used in last 20 seconds
+    if !@usedproxies[chosen] || @usedproxies[chosen] < Time.now-20
+      @usedproxies[chosen] = Time.now
+      return chosen
+    else
+      sleep(0.5)
+      getRandomProxy
+    end
+  end
+end

metadata

@@ -1,22 +1,24 @@
 --- !ruby/object:Gem::Specification
 name: generalscraper
 version: !ruby/object:Gem::Version
-  version: 0.0.3
+  version: 0.0.4
 platform: ruby
 authors:
 - M. C. McGrath
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-05-18 00:00:00.000000000 Z
+date: 2014-04-07 00:00:00.000000000 Z
 dependencies: []
-description: Scrapes all pages on a site you specify including terms you specify.
+description: Scrapes Google
 email: shidash@shidash.com
 executables: []
 extensions: []
 extra_rdoc_files: []
 files:
 - lib/generalscraper.rb
+- lib/parse_page.rb
+- lib/proxy_manager.rb
 homepage: https://github.com/TransparencyToolkit/generalscraper
 licenses:
 - GPL
@@ -27,19 +29,18 @@ require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.0.14
+rubygems_version: 2.4.6
 signing_key: 
 specification_version: 4
-summary: Get all pages on a site for terms specified
+summary: Scrapes Google
 test_files: []
-has_rdoc: