generalscraper 0.0.23 → 0.0.24

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 85b3d91e96159d5f3cd36961664721d9bd5e7313
-  data.tar.gz: 7568e30d7343d9be690e48e0369f7cb3db194a81
+  metadata.gz: c14621c2ab26d98c1a4f8998a47de2454c6bbfad
+  data.tar.gz: d388fab99cffa9ec89e283428e85015c545b9c1e
 SHA512:
-  metadata.gz: b88b2d814a08bc24b68ed337e4e973471a57c02d1fa9323156ccd7a93f5dab754dde734e48326378e203e5c11599ee1e0be789d45eb728016feb194b0949094b
-  data.tar.gz: 5f32fd7d6da2aea69a4654a2e3e3662bcbce88412252a4b366048ffe14d02e0d33a7f57b653138b9de9eb1a7f2117cf38023072b8936a9cbd10e0e5ed7f001b7
+  metadata.gz: a6b5058cb399dc7e5deb8e9fc2b5ea6ba9097ad9339f32c470a2e58c95688ca261a05769c6dce3c1a85cd93b9a884bd458e8737f11a5c9343839048fcd116a7e
+  data.tar.gz: 0700a3c861858921cba2ff45867a7afc33bd2233a748e4a0e9e98a09aa1b061a453edd97a6ddaaf5f3e73ec5209fecea80771143e846d512d0f78be8e70ec866

data/lib/generalscraper.rb

@@ -30,6 +30,7 @@ class GeneralScraper
   def search
     check_results(@requests.get_page("http://google.com", @operators + " " + @searchterm),
                   "http://google.com", (@operators + " " + @searchterm))
+    report_status("Got search results for " + @operators.to_s + " " + @searchterm.to_s)
   end
 
   # Check that page with links loaded
@@ -45,13 +46,15 @@ class GeneralScraper
         check_results(@requests.get_updated_current_page)
         
       else # Restart and try again if CAPTCHA-solving not enabled
+        report_status("CAPTCHA Found. CAPTCHA solving not enabled. Trying to restart browser.")
         @requests.restart_browser
         check_results(@requests.get_page(requested_page), requested_page)
       end
     else # No CAPTCHA found :)
       begin
         navigate_save_results(page)
-      rescue Exception
+      rescue => e
+        report_status("Error: " + e.to_s + " Retrying...")
         @requests.restart_browser
         check_results(@requests.get_page(requested_page), requested_page)
       end
@@ -66,8 +69,8 @@ class GeneralScraper
     return yield(html).inject(Array.new) do |link_arr, al|
       begin
         link_arr.push(al["href"])
-      rescue
-        
+      rescue => e
+        report_status("Error getting links: " + e.to_s)
       end
      
       link_arr
@@ -85,6 +88,7 @@ class GeneralScraper
     # Go to next page
     next_pages = get_links(page) {|html| html.css("#pnnext")}
     next_pages.each do |link|
+      report_status("Going to next page: google.com"+link)
       next_search_page("google.com"+link)
     end
   end
@@ -114,6 +118,7 @@ class GeneralScraper
       end
     end
 
+    report_status("Finished collecting data for " + @operators.to_s + " " + @searchterm.to_s)
     @requests.close_all_browsers
   end
 
@@ -135,6 +140,16 @@ class GeneralScraper
                              Curl::PostField.content('results', JSON.pretty_generate([results])))
   end
 
+  # Report Harvester status message
+  def report_status(status_msg)
+    if @cm_url
+      curl_url = @cm_url+"/update_status"
+      c = Curl::Easy.http_post(curl_url,
+                               Curl::PostField.content('selector_id', @selector_id),
+                               Curl::PostField.content('status_message', status_msg))
+    end
+  end
+  
   # Add page hash to output for bulk reporting
   def report_bulk(results)
     @output.push(results)

data/lib/parse_page.rb

@@ -16,6 +16,7 @@ module ParsePage
       begin
         return getPDF(url, pagehash)
       rescue
+        report_status("PDF parsing failed for "+url.to_s)
         return nil
       end
     else

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: generalscraper
 version: !ruby/object:Gem::Version
-  version: 0.0.23
+  version: 0.0.24
 platform: ruby
 authors:
 - M. C. McGrath
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-10-08 00:00:00.000000000 Z
+date: 2016-10-30 00:00:00.000000000 Z
 dependencies: []
 description: Scrapes Google
 email: shidash@shidash.com