generalscraper 0.0.9 → 0.0.10

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 6d547a3d1293c92e7f9553422668a0fb0f382b3a
-  data.tar.gz: 4f060e26095d1abc162d348c1a5f3aa0c334d32b
+  metadata.gz: 29d8ecb9bacae3fb5f21d57bd48aa6f7aa7bf9e3
+  data.tar.gz: 7bb42fc8560c5be6e65dd93827b4eafbfa647718
 SHA512:
-  metadata.gz: 96d5d92f5f376dbbd8a7d42a4e6cb0bffc97768a261fe9d59afd3b56302d5bce98cdc452ad7eebdbcbb540216e459562252ce372ba3419d36df508e9dcd45b06
-  data.tar.gz: 3a2c4a317a140f6183d0ac9baf2e6cafd1b4777b648f0cc84805a20ba40f87aa8fc932126dca5f93641482df9486e3182073888fd905263d56d3e4dabfbcfce2
+  metadata.gz: 02e934121567d5fb9f18392581ac145e94f01ee1473261090d15fd0490abeff8a96888fed556f792e46728725d6d03aa542fb34fc309e0dc66ecc2204d6988c7
+  data.tar.gz: c020dcf77f83b20e52325988ad6401a8e315410bbda70d4b5d58842065579cb897bfe6869c9f88eb8e60a89f243835bd4fbbdd5fa4713910b77033684d4b3840

data/lib/generalscraper.rb

@@ -20,48 +20,75 @@ class GeneralScraper
     @urllist = Array.new
     @startindex = 10
     @use_proxy = use_proxy
+
+    # Generate driver
+    profile = Selenium::WebDriver::Firefox::Profile.new
+    profile['intl.accept_languages'] = 'en'
+    @driver = Selenium::WebDriver.for :firefox, profile: profile
   end
 
   # Searches for links on Google
   def search
-    categorizeLinks(getPage("http://google.com", @operators + " " + @searchterm, @use_proxy))
+    categorizeLinks(getPage("http://google.com", @driver, @operators + " " + @searchterm, @use_proxy))
+  end
+
+  # Gets the links from the page
+  def getLinks(page)
+    # Sleep while things load
+    sleep(10)
+    
+    # Extract arr
+    return page.find_elements(css: "a").inject(Array.new) do |link_arr, al|
+      begin
+        link_arr.push(al.attribute("href"))
+      rescue
+        
+      end
+     
+      link_arr
+    end
   end
 
   # Categorizes the links on results page into results and other search pages
   def categorizeLinks(page)
-    page.links.each do |link|
-      if isResultLink?(link)
-        siteURLSave(link)
-      elsif isSearchPageLink?(link)
-        nextSearchPage(link)
+    links = getLinks(page)
+    links.each do |link| 
+      if link
+        if isResultLink?(link)
+          siteURLSave(link)
+        elsif isSearchPageLink?(link)
+          nextSearchPage(link)
+        end
       end
     end
   end
 
   # Determines if url is link to search result
   def isResultLink?(link)
-    return (link.href.include? @op_val) && (!link.href.include? "webcache") && (!link.href.include? @operators.gsub(" ", "+"))
+    return (link.include? @op_val) &&
+           (!link.include? "webcache") &&
+           (!link.include? @operators.gsub(" ", "+")) &&
+           (!link.include?("translate.google"))
   end
 
   # Determines if URL is link to next search page
   def isSearchPageLink?(link)
-    return (link.href.include? "&sa=N") && (link.href.include? "&start=")
+    return (link.include? "&sa=N") && (link.include? "&start=")
   end
 
   
   # Parse and save the URLs for search results
   def siteURLSave(link)
-    site_url = link.href.split("?q=")[1]
-    @urllist.push(site_url.split("&")[0]) if site_url
+    @urllist.push(link)
   end
 
   # Process search links and go to next page
   def nextSearchPage(link)
-    page_index_num = link.href.split("&start=")[1].split("&sa=N")[0]
-    
+    page_index_num = link.split("&start=")[1].split("&sa=N")[0]
+  
     if page_index_num.to_i == @startindex
       @startindex += 10
-      categorizeLinks(getPage("http://google.com" + link.href + "&filter=0", @use_proxy))
+      categorizeLinks(getPage(link, @driver, @use_proxy))
     end
   end
 
@@ -70,14 +97,17 @@ class GeneralScraper
   def getData
     search
     @urllist.each do |url|
-      getPageData(url)
+      getPageData(url, @driver)
     end
+    @driver.close
     return JSON.pretty_generate(@output)
   end
 
   # Returns a list of search result URLs
   def getURLs
     search
+    @driver.close
     return JSON.pretty_generate(@urllist)
   end
 end
+

data/lib/parse_page.rb

@@ -2,9 +2,10 @@ require 'uploadconvert'
 
 module ParsePage
   # Get both page metadata and text
-  def getPageData(url)
+  def getPageData(url, driver)
     begin
-      html = Nokogiri::HTML(getPage(url).body)
+      page = getPage(url, driver, nil, 5, false)
+      html = Nokogiri::HTML(page.page_source)
       pagehash = getMetadata(url, html)
       pagehash = getContent(url, pagehash, html)
       @output.push(pagehash)

data/lib/proxy_manager.rb

@@ -1,10 +1,11 @@
 require 'active_support/time'
 require 'mechanize'
 require 'uri'
+require 'selenium-webdriver'
 
 module ProxyManager 
   # Get the page with a proxy
-  def getPage(url, form_input = nil, fail_count = 0, use_proxy)
+  def getPage(url, driver, form_input = nil, fail_count = 0, use_proxy)
     agent = Mechanize.new do |a|
       a.user_agent_alias = "Linux Firefox"
 
@@ -19,12 +20,17 @@ module ProxyManager
     # Slightly different based on filling in form or not
     begin
       if form_input
-        gform = agent.get(url).form("f")
-        gform.q = form_input
-        return agent.submit(gform, gform.buttons.first)
+        driver.navigate.to url
+        element = driver.find_element(name: "q")
+        element.send_keys form_input
+        element.submit
+        puts "Searched for: " + form_input
+        
+        return driver
       else
         puts "Getting page " + url
-        return agent.get(url)
+        driver.navigate.to url
+        return driver
       end
     rescue # Only retry request 10 times
       puts "FAILED"

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: generalscraper
 version: !ruby/object:Gem::Version
-  version: 0.0.9
+  version: 0.0.10
 platform: ruby
 authors:
 - M. C. McGrath