generalscraper 0.0.11 → 0.0.12

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 338629c39c095b6f64780ab9d13f5372234f8ed8
-  data.tar.gz: ae8586f85b7c14cb1b6e0e44dd7eb90b590d6e8c
+  metadata.gz: a23d6483229cb3d18a14d8ba906658779f1a80d2
+  data.tar.gz: 0a719a46f8b3091880419495ad8e746d3e53e722
 SHA512:
-  metadata.gz: 7d3c7370d236d984f0a6e49790f1bda765ad817b1d6a3680d90d8c682c992aeeb7723e64d7a4c0d2f1e761b5403851313f814d1537d4741ba7d75951305fee28
-  data.tar.gz: a2dd8f531f2a9fbcc61bc0f206765da233fead65bfa0ac0e045472c0af465e7758d822e1b0bacea473498ee7e45caa6492223eadc1cd5d9d9c19ffa3329a572c
+  metadata.gz: 9554472410fb5879f28de9fdd3e55625c2d5016435bf3df4ad314d6f2cb9b77101fde746f97d651575e006f49d3825256095ea6b73d1154d5072d5cee8ee12c1
+  data.tar.gz: 76f39348dde21f700ce560346243b5b18e857340ce59a973cb23c6735d89ea626d93a3e1cd0d3c6027e201e74d2fef1762a640f0c3cbb35792f89c5300def338

data/lib/generalscraper.rb

@@ -1,46 +1,50 @@
 require 'json'
 require 'nokogiri'
 require 'mechanize'
+require 'requestmanager'
+require 'pry'
 
 load 'parse_page.rb'
-load 'proxy_manager.rb'
 
 class GeneralScraper
   include ParsePage
-  include ProxyManager
   
-  def initialize(operators, searchterm, proxylist, use_proxy)
+  def initialize(operators, searchterm, proxylist)
     @operators = operators
     @searchterm = searchterm
     @op_val = @operators.split(" ")[0].split(":")[1]
-    @proxylist = IO.readlines(proxylist)
-    @usedproxies = Hash.new
+    @proxylist = proxylist
+    @requests = RequestManager.new(@proxylist, [4, 15], 1)
     
     @output = Array.new
     @urllist = Array.new
     @startindex = 10
-    @use_proxy = use_proxy
-
-    # Generate driver
-    profile = Selenium::WebDriver::Firefox::Profile.new
-    profile['intl.accept_languages'] = 'en'
-    @driver = Selenium::WebDriver.for :firefox, profile: profile
   end
 
   # Searches for links on Google
   def search
-    categorizeLinks(getPage("http://google.com", @driver, @operators + " " + @searchterm, @use_proxy))
+    check_results(@requests.get_page("http://google.com", @operators + " " + @searchterm),
+                  "http://google.com", (@operators + " " + @searchterm))
+  end
+
+  # Check that page with links loaded
+  def check_results(page, *requested_page)
+    if page.include?("To continue, please type the characters below:")
+      @requests.restart_browser
+      check_results(@requests.get_page(requested_page), requested_page)
+    else
+      categorizeLinks(page)
+    end
   end
 
   # Gets the links from the page
-  def getLinks(page)
-    # Sleep while things load
-    sleep(10)
-    
-    # Extract arr
-    return page.find_elements(css: "a").inject(Array.new) do |link_arr, al|
+  def getLinks(page)   
+    html = Nokogiri::HTML(page)
+
+    # Get array of links
+    return html.css("a").inject(Array.new) do |link_arr, al|
       begin
-        link_arr.push(al.attribute("href"))
+        link_arr.push(al["href"])
       rescue
         
       end
@@ -52,12 +56,14 @@ class GeneralScraper
   # Categorizes the links on results page into results and other search pages
   def categorizeLinks(page)
     links = getLinks(page)
+
+    # Categorize as results or search pages
     links.each do |link| 
       if link
         if isResultLink?(link)
           siteURLSave(link)
         elsif isSearchPageLink?(link)
-          nextSearchPage(link)
+          nextSearchPage("google.com"+link)
         end
       end
     end
@@ -88,26 +94,25 @@ class GeneralScraper
   
     if page_index_num.to_i == @startindex
       @startindex += 10
-      categorizeLinks(getPage(link, @driver, @use_proxy))
+      check_results(@requests.get_page(link), link)
     end
   end
 
-  
   # Gets all data and returns in JSON
   def getData
     search
     @urllist.each do |url|
-      getPageData(url, @driver)
+      getPageData(url)
     end
-    @driver.close
+
+    @requests.close_all_browsers
     return JSON.pretty_generate(@output)
   end
 
   # Returns a list of search result URLs
   def getURLs
     search
-    @driver.close
+    @requests.close_all_browsers
     return JSON.pretty_generate(@urllist)
   end
 end
-

data/lib/parse_page.rb

@@ -2,10 +2,10 @@ require 'uploadconvert'
 
 module ParsePage
   # Get both page metadata and text
-  def getPageData(url, driver)
+  def getPageData(url)
     begin
-      page = getPage(url, driver, nil, 5, false)
-      html = Nokogiri::HTML(page.page_source)
+      page = @requests.get_page(url)
+      html = Nokogiri::HTML(page)
       pagehash = getMetadata(url, html)
       pagehash = getContent(url, pagehash, html)
       @output.push(pagehash)

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: generalscraper
 version: !ruby/object:Gem::Version
-  version: 0.0.11
+  version: 0.0.12
 platform: ruby
 authors:
 - M. C. McGrath
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-10-21 00:00:00.000000000 Z
+date: 2015-11-03 00:00:00.000000000 Z
 dependencies: []
 description: Scrapes Google
 email: shidash@shidash.com
@@ -18,7 +18,6 @@ extra_rdoc_files: []
 files:
 - lib/generalscraper.rb
 - lib/parse_page.rb
-- lib/proxy_manager.rb
 homepage: https://github.com/TransparencyToolkit/generalscraper
 licenses:
 - GPL
@@ -39,8 +38,9 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.4.6
+rubygems_version: 2.4.8
 signing_key: 
 specification_version: 4
 summary: Scrapes Google
 test_files: []
+has_rdoc: 

data/lib/proxy_manager.rb

@@ -1,70 +0,0 @@
-require 'active_support/time'
-require 'mechanize'
-require 'uri'
-require 'selenium-webdriver'
-
-module ProxyManager 
-  # Get the page with a proxy
-  def getPage(url, driver, form_input = nil, fail_count = 0, use_proxy)
-    agent = Mechanize.new do |a|
-      a.user_agent_alias = "Linux Firefox"
-
-      # Set proxy if specified, otherwise delay to avoid blocks
-      if use_proxy
-        a.set_proxy(*getRandomProxy(url))
-      else
-        sleep(rand(30..90))
-      end
-    end
-
-    # Slightly different based on filling in form or not
-    begin
-      if form_input
-        driver.navigate.to url
-        element = driver.find_element(name: "q")
-        element.send_keys form_input
-        element.submit
-        puts "Searched for: " + form_input
-        
-        return driver
-      else
-        puts "Getting page " + url
-        driver.navigate.to url
-        return driver
-      end
-    rescue # Only retry request 10 times
-      begin
-        puts "FAILED"
-        getPage(url, form_input, fail_count+=1) if fail_count < 10
-      rescue
-      end
-    end
-  end
-
-  # Choose a random proxy
-  def getRandomProxy(url)
-    max = @proxylist.length
-    chosen = @proxylist[Random.rand(max)]
-
-    # Only use proxy if it hasn't been used in last 20 seconds on same host
-    if isNotUsed?(chosen, url)
-      @usedproxies[chosen] = [Time.now, URI.parse(url).host]
-      return parseProxy(chosen)
-    else
-      sleep(0.005)
-      getRandomProxy(url)
-    end
-  end
-
-  # Splits up proxy into IP, port, user, password
-  def parseProxy(chosen)
-    proxy_info = chosen.split(":")
-    proxy_info[proxy_info.length-1] = proxy_info.last.strip
-    return proxy_info
-  end
-
-  # Checks if a proxy has been used on domain in the last 20 seconds
-  def isNotUsed?(chosen, url)
-    return !@usedproxies[chosen] || @usedproxies[chosen][0] <= Time.now-20 || @usedproxies[chosen][1] != URI.parse(url).host
-  end
-end