linkedincrawler 0.0.18 → 0.0.19

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 0256d272cfb618594d7bc0bb926252d890f86101
-  data.tar.gz: f1a370dcc7e4b28dda00b885a66496bc31d9d733
+  metadata.gz: b55cee51c70158577d2c1e1c618aa284f6a4647c
+  data.tar.gz: 7c2ab05da2829fb7f85f6510a3eee45ba7ef65df
 SHA512:
-  metadata.gz: a134b51328467c71c7e3fe709fb59ed290380244aa77015539eeeff84c23468f5afd8da7964f2a09ee695b81b2832569d7c9f8fcbe7b3da4d8d8e44428920588
-  data.tar.gz: 93ae61b064aa8bdd24beaa8e07c5e0b4fed0943a18c689907c03cb997db43891990f5be4c1c97c7e7696a99aeda536faf7be37cd899a36bdb63391b5dc2f3457
+  metadata.gz: 51c17a2cd6eb7a819a095291aa84fc52cbb7ddf00e7fbb0c627f9dd19a10181c86f195a2b807253ad89820761e0025eed2c07865b184d763d4a94359f7fad077
+  data.tar.gz: 9f9f9ef9af43157daac01b2a2e085782538eb5441f46c98f754f26c9428260b0193df59125cc8dd8bfa352a7b46b88a79c39279ce5666aabc2e95afdf71be652

data/lib/linkedincrawler.rb

@@ -19,22 +19,24 @@ class LinkedinCrawler
     @solver_details = solver_details
 
     # Handle crawler manager info
+    @cm_hash = cm_hash
     @cm_url = cm_hash[:crawler_manager_url] if cm_hash
     @selector_id = cm_hash[:selector_id] if cm_hash
   end
 
   # Run search terms and get results
   def search
-
+   
     begin
       # Run Google search
-    g = GeneralScraper.new("site:linkedin.com/pub -site:linkedin.com/pub/dir/", @search_terms, @requests_google, @solver_details, nil)
+    g = GeneralScraper.new("site:linkedin.com/pub -site:linkedin.com/pub/dir/", @search_terms, @requests_google, @solver_details, @cm_hash)
     urls = g.getURLs
    
     # Look for new LI urls
-    g2 = GeneralScraper.new("site:linkedin.com/in", @search_terms, @requests_google2, @solver_details, nil)
+    g2 = GeneralScraper.new("site:linkedin.com/in", @search_terms, @requests_google2, @solver_details, @cm_hash)
     urls = JSON.parse(urls) + JSON.parse(g2.getURLs)
-    rescue Exception
+    rescue => e
+      report_status("Error running Google Crawler from LinkedIn Crawler: " +e.to_s)
       binding.pry
     end
     
@@ -47,6 +49,7 @@ class LinkedinCrawler
 
     # Close all the browsers when done
     @requests.close_all_browsers
+    report_status("Data collection completed for " + @search_terms.to_s)
   end
 
   # Check that it is actually a LinkedIn profile page
@@ -84,8 +87,10 @@ class LinkedinCrawler
       if @retry_count < @retry_limit
         @requests.restart_browser
         @retry_count += 1
+        report_status("Profile parsing failed for "+profile_url.to_s+". Retrying...")
         scrape(profile_url)
       else # Just save it and move on
+        report_status("Profile parsing failed for "+profile_url.to_s+". Moving on.")
         report_results(parsed_profile, profile_url)
       end
       
@@ -113,6 +118,16 @@ class LinkedinCrawler
                              Curl::PostField.content('results', JSON.pretty_generate(results)))
   end
 
+  # Report Harvester status message
+  def report_status(status_msg)
+    if @cm_url
+      curl_url = @cm_url+"/update_status"
+      c = Curl::Easy.http_post(curl_url,
+                               Curl::PostField.content('selector_id', @selector_id),
+                               Curl::PostField.content('status_message', status_msg))
+    end
+  end
+
   # Print output in JSON
   def gen_json
     JSON.pretty_generate(@output)

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: linkedincrawler
 version: !ruby/object:Gem::Version
-  version: 0.0.18
+  version: 0.0.19
 platform: ruby
 authors:
 - M. C. McGrath
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-10-08 00:00:00.000000000 Z
+date: 2016-10-30 00:00:00.000000000 Z
 dependencies: []
 description: Crawls public LinkedIn profiles via Google
 email: shidash@shidash.com