linkedincrawler 0.0.13 → 0.0.14

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: b0d3e42277b83b79704ceaa5d7b07a0a1996b5d2
-  data.tar.gz: c2189d0a40e02f8f0e6e085720cc60335cdb2cef
+  metadata.gz: 9cab546fac1482b262f2435257af20d645b7a2ca
+  data.tar.gz: 8a18d774dbaf1791b13a104b85bd96edd3400dc9
 SHA512:
-  metadata.gz: db465540122c5ec210afacec2c3445827542596c2db78271a3eef97e18dbd9795e6198c8cf7d0632838c77cbf197fbf127709f3cbd3902aacd4f2cdb3c04a616
-  data.tar.gz: f5e34bd6e608e8c675029c353c7379ac2b5af1433ac9047f1cf8708c0a0adfe72a145ec99333ab71d01f3150a95ca15667f7f69052d1147c02b8a6ca6a670ccc
+  metadata.gz: fb059768cdaf5204cfef18ca7fc8ec8670c0ed4bed5346fc984ab57a44eb6d7fa988e8e2b584920b1f500a20257196fcf45bedaafdf86197887e3ad232fe398c
+  data.tar.gz: e6b09adb74fec01250746172e372e82b5277badda83c8156d8c60049945d36bdafcc528e5bf69f2063c219cfc3508c771138d363a0226742ac45aa25a0eb00e7

data/lib/linkedincrawler.rb

@@ -44,25 +44,39 @@ class LinkedinCrawler
            @retry_count < @retry_limit
   end
 
+  # Add the parsed profile to output, reset the retry count, and continue
+  def save_and_continue(parsed_profile)
+    @output += parsed_profile if parsed_profile != nil && !parsed_profile.empty?
+    @retry_count = 0
+  end
+
+  # Check if profile parsed successfully
+  def profile_parsing_failed?(parsed_profile)
+    return (parsed_profile == nil) || parsed_profile.empty? || parsed_profile.first["parsing_failed"]
+  end
+
   # Scrape each page
   def scrape(profile_url)
     # Get profile page
     profile_html = @requests.get_page(profile_url)
 
-    # Parse profile and add to output
-    begin
-      l = LinkedinParser.new(profile_html, profile_url, {timestamp: Time.now, search_terms: @search_terms})
-      @output += JSON.parse(l.results_by_job)
-      @retry_count = 0
-    rescue
-      # If proxy doesn't work, try another a few times
-      if check_right_page(profile_url)
+    # Parse profile
+    l = LinkedinParser.new(profile_html, profile_url, {timestamp: Time.now, search_terms: @search_terms})
+    parsed_profile = JSON.parse(l.results_by_job)
+
+    # Check if it failed or succeeded
+    if profile_parsing_failed?(parsed_profile)
+      # Handle something wrong- restart in case it is blocked and rescrape
+      if @retry_count < @retry_limit
         @requests.restart_browser
         @retry_count += 1
         scrape(profile_url)
-      else
-        @retry_count = 0
+      else # Just save it and move on
+        save_and_continue(parsed_profile)
       end
+      
+    else # It succeeded!
+      save_and_continue(parsed_profile)
     end
   end
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: linkedincrawler
 version: !ruby/object:Gem::Version
-  version: 0.0.13
+  version: 0.0.14
 platform: ruby
 authors:
 - M. C. McGrath
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-12-05 00:00:00.000000000 Z
+date: 2015-12-06 00:00:00.000000000 Z
 dependencies: []
 description: Crawls public LinkedIn profiles via Google
 email: shidash@shidash.com