linkedincrawler 0.0.15 → 0.0.16

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: d133e662b783dcf61594013b375ddec17cb22102
-  data.tar.gz: 4c477ee0f284c4bc13171d25c747cf9f2b42c090
+  metadata.gz: e497521e3f492f55300fe474d802e36645cb9ff3
+  data.tar.gz: 1c7599985292c36c758577b0325dfa49fa671ec7
 SHA512:
-  metadata.gz: 47a121927417ce0d195d5be681d533f35c88613710f104d540e032bc9539ebd25840163eda54c75a63caeb27924f1fc65fd767042e5e18664d4c0fd13809e01e
-  data.tar.gz: 87f53246bb4d98c67683525ea32095729a453502e0426f87268edd949809fea93ea27da0f288598a1698202b433e7a75f660c64820a2bb66b2605a56ad8d563c
+  metadata.gz: a7b2616237182ca01b2144ae3b527c27c7adb76f41d454b6cb01f7ae7e10151cbe519815aadeb838ecbb1d0f9ae247829b584bbf74d8c69364cda0d7b262481f
+  data.tar.gz: 4911139eb248197b6bf503fe5d3373beee04e3033eae8283106fa4141e3911a938f353190e1ffbe92abc633f184b3c401239c3208469a4f2c13bd36e1166efda

data/lib/linkedincrawler.rb

@@ -6,7 +6,7 @@ require 'selenium-webdriver'
 require 'pry'
 
 class LinkedinCrawler
-  def initialize(search_terms, retry_limit, requests, requests_google, requests_google2, solver_details)
+  def initialize(search_terms, retry_limit, requests, requests_google, requests_google2, solver_details, cm_hash)
     @search_terms = search_terms
     @output = Array.new
     
@@ -17,10 +17,16 @@ class LinkedinCrawler
     @requests_google = requests_google
     @requests_google2 = requests_google2
     @solver_details = solver_details
+
+    # Handle crawler manager info
+    @cm_url = cm_hash[:crawler_manager_url] if cm_hash
+    @selector_id = cm_hash[:selector_id] if cm_hash
   end
 
   # Run search terms and get results
   def search
+
+    begin
     # Run Google search
     g = GeneralScraper.new("site:linkedin.com/pub -site:linkedin.com/pub/dir/", @search_terms, @requests_google, @solver_details)
     urls = g.getURLs
@@ -28,6 +34,9 @@ class LinkedinCrawler
     # Look for new LI urls
     g2 = GeneralScraper.new("site:linkedin.com/in", @search_terms, @requests_google2, @solver_details)
     urls = JSON.parse(urls) + JSON.parse(g2.getURLs)
+    rescue Exception
+      binding.pry
+    end
     
     # Scrape each resulting LinkedIn page
     urls.each do |profile|
@@ -77,14 +86,33 @@ class LinkedinCrawler
         @retry_count += 1
         scrape(profile_url)
       else # Just save it and move on
-        save_and_continue(parsed_profile)
+        report_results(parsed_profile, profile_url)
       end
       
     else # It succeeded!
-      save_and_continue(parsed_profile)
+      report_results(parsed_profile, profile_url)
     end
   end
 
+  # Figure out how to report results
+  def report_results(results, link)
+    if @cm_url
+      report_incremental(results, link)
+    else
+      save_and_continue(results)
+    end
+  end
+
+  # Report results back to Harvester incrementally
+  def report_incremental(results, link)
+    curl_url = @cm_url+"/relay_results"
+    @retry_count = 0
+    c = Curl::Easy.http_post(curl_url,
+                             Curl::PostField.content('selector_id', @selector_id),
+                             Curl::PostField.content('status_message', "Collected " + link),
+                             Curl::PostField.content('results', JSON.pretty_generate(results)))
+  end
+
   # Print output in JSON
   def gen_json
     JSON.pretty_generate(@output)

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: linkedincrawler
 version: !ruby/object:Gem::Version
-  version: 0.0.15
+  version: 0.0.16
 platform: ruby
 authors:
 - M. C. McGrath
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-03-16 00:00:00.000000000 Z
+date: 2016-10-07 00:00:00.000000000 Z
 dependencies: []
 description: Crawls public LinkedIn profiles via Google
 email: shidash@shidash.com