generalscraper 0.0.19 → 0.0.20

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 9de0cd6461d23d2f27b9d7ced18698f503c481aa
-  data.tar.gz: 3ed9dd6ecf0b84e8c31a3002a0e6e83689848208
+  metadata.gz: ae1747657ed5aed8d784aee72c2203ffa8d0c5b9
+  data.tar.gz: e9e0e82429b6085d0455b8425123aca817229839
 SHA512:
-  metadata.gz: 678789d0b479ceba78cf9b464b21150e1d2469da95891043f0996f530f3a957c34ef02bc0c31458835a39170980190c1cac478fe44652d301b082e58ad1b103d
-  data.tar.gz: 9196461845feecbafa876cd29cafde6b60ae1269a8eba3ada28fa5e7b778a95324d0176fd8488f944c3bbe7bdd8959e0db91a128cedcc004218a12f4aebf6cc0
+  metadata.gz: 9a97172a7739666794cc170973f5433b16fad53bb7ed4993e3e15d05910c650b13202bc5a053cbc3642f9d17b95a20fdc4c75306bbee9784cb209d51188f2d9e
+  data.tar.gz: 36ea20ed630476467fd7a4cecec499e4d6aa98e03b7a0335a1e4ac9329eecc88f008b1813e546f36c0d561fc5ee6f2de34dcd92509b45458c4adf26aa9dc3802

data/lib/generalscraper.rb

@@ -10,7 +10,7 @@ load 'captcha.rb'
 class GeneralScraper
   include ParsePage
   
-  def initialize(operators, searchterm, requests, solver_details)
+  def initialize(operators, searchterm, requests, solver_details, cm_hash)
     @operators = operators
     @searchterm = searchterm
     @op_val = @operators.split(" ")[0].split(":")[1]
@@ -20,6 +20,10 @@ class GeneralScraper
     @output = Array.new
     @urllist = Array.new
     @startindex = 10
+
+    # Handle crawler manager info
+    @cm_url = cm_hash[:crawler_manager_url] if cm_hash
+    @selector_id = cm_hash[:selector_id] if cm_hash
   end
 
   # Searches for links on Google
@@ -44,11 +48,13 @@ class GeneralScraper
         @requests.restart_browser
         check_results(@requests.get_page(requested_page), requested_page)
       end
-    elsif page.include?("403") && page.length < 100
-      @requests.restart_browser
-      check_results(@requests.get_page(requested_page), requested_page)
     else # No CAPTCHA found :)
-      navigate_save_results(page)
+      begin
+        navigate_save_results(page)
+      rescue Exception
+        @requests.restart_browser
+        check_results(@requests.get_page(requested_page), requested_page)
+      end
     end
   end
 
@@ -102,17 +108,46 @@ class GeneralScraper
   def getData
     search
     @urllist.each do |url|
-      getPageData(url)
+      report_results(getPageData(url), url)
     end
 
     @requests.close_all_browsers
-    return JSON.pretty_generate(@output)
   end
 
+  # Figure out how to report results
+  def report_results(results, link)
+    if @cm_url
+      report_incremental(results, link)
+    else
+      report_bulk(results)
+    end
+  end
+
+  # Report results back to Harvester incrementally
+  def report_incremental(results, link)
+    curl_url = @cm_url+"/relay_results"
+    c = Curl::Easy.http_post(curl_url,
+                             Curl::PostField.content('selector_id', @selector_id),
+                             Curl::PostField.content('status_message', "Collected " + link),
+                             Curl::PostField.content('results', JSON.pretty_generate(results)))
+  end
+
+  # Add page hash to output for bulk reporting
+  def report_bulk(results)
+    @output.push(results)
+  end
+  
+
   # Returns a list of search result URLs
   def getURLs
     search
     @requests.close_all_browsers
     return JSON.pretty_generate(@urllist)
   end
+
+  # Get the JSON of all the data
+  def get_json_data
+    return JSON.pretty_generate(@output)
+  end
 end
+

data/lib/parse_page.rb

@@ -8,7 +8,7 @@ module ParsePage
       html = Nokogiri::HTML(page)
       pagehash = getMetadata(url, html)
       pagehash = getContent(url, pagehash, html)
-      @output.push(pagehash)
+      return pagehash
     rescue
     end
   end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: generalscraper
 version: !ruby/object:Gem::Version
-  version: 0.0.19
+  version: 0.0.20
 platform: ruby
 authors:
 - M. C. McGrath
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-03-16 00:00:00.000000000 Z
+date: 2016-10-08 00:00:00.000000000 Z
 dependencies: []
 description: Scrapes Google
 email: shidash@shidash.com