linkedincrawler 0.0.10 → 0.0.11

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: ad1998e603add9f4dc968ade37d9eec959f0135f
-  data.tar.gz: fbf79531d15ddd6584ca4a4e34b10d1ce33ce6d1
+  metadata.gz: 64048a6ad5a246281c7546bd04105db4a4d2d8de
+  data.tar.gz: e933a51855e800a285b90e39e8129a146aa17e5e
 SHA512:
-  metadata.gz: cdaf3cdd5ced3d72ef91cbc151493616e2f1ae7258ff5e2018a35363128f1778f6c4350394b1e53518c7f9748af861c2eb259f882d5b7ab2c4b11c6b53390c01
-  data.tar.gz: 8eab640479ba0f4294e5eaf169b9fabe26f64f11aec04450e1ea2b0245fa3d9e0c51d62d0a586ec39b65870b5a7294df7f853ee39785f656c89f5f0458625d44
+  metadata.gz: 4b533610302c151219fe9d5f3fb1c1364b58bacb696c9b2d837c7a826bd735482de5328a64cfe8c97cf6888375317726dd7b70f70229c9148bde0cbc825e4d2f
+  data.tar.gz: f4c8663a5622ca92047039e14ddf1de69dab44e7541481ea37069316aa9f4fd78048ec84da2d9ac48a58ce5344c8cdfe03ce6a97ea13434ae5e60315566501ea

data/lib/linkedincrawler.rb

@@ -6,44 +6,38 @@ require 'selenium-webdriver'
 require 'pry'
 
 class LinkedinCrawler
-  def initialize(search_terms, retry_limit, proxy_list, request_time)
+  def initialize(search_terms, retry_limit, requests, requests_google)
     @search_terms = search_terms
     @output = Array.new
+    
     @retry_limit = retry_limit
     @retry_count = 0
-    @proxy_list = proxy_list
-    @requests = RequestManager.new(@proxy_list, request_time, 5)
+    
+    @requests = requests
+    @requests_google = requests_google
   end
 
   # Run search terms and get results
   def search
     # Run Google search
-    g = GeneralScraper.new("site:linkedin.com/pub -site:linkedin.com/pub/dir/", @search_terms, nil)
-   # begin
-      urls = g.getURLs
-   # rescue # Search again if it didn't work the first time
-    #  search
-    #end
+    g = GeneralScraper.new("site:linkedin.com/pub -site:linkedin.com/pub/dir/", @search_terms, @requests_google)
+    urls = g.getURLs
 
-    # Search again if it didn't run
-   # if urls.length == 0 || urls.empty?
-#      search
-   # else
-      # Scrape each resulting LinkedIn page
-      JSON.parse(urls).each do |profile|
-        if profile.include?(".linkedin.") && !profile.include?("/search")
-          scrape(profile)
-        end
+    # Scrape each resulting LinkedIn page
+    JSON.parse(urls).each do |profile|
+      if check_right_page(profile)
+        scrape(profile)
       end
-   # end
+    end
 
-    # Close all the browsers
+    # Close all the browsers when done
     @requests.close_all_browsers
   end
 
   # Check that it is actually a LinkedIn profile page
   def check_right_page(profile_url)
     return !profile_url.include?("www.google") &&
+           profile_url.include?(".linkedin.") &&
            !profile_url.include?("linkedin.com/pub/dir") &&
            !profile_url.include?("/search") &&
            @retry_count < @retry_limit
@@ -65,6 +59,8 @@ class LinkedinCrawler
         @requests.restart_browser
         @retry_count += 1
         scrape(profile_url)
+      else
+        @retry_count = 0
       end
     end
   end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: linkedincrawler
 version: !ruby/object:Gem::Version
-  version: 0.0.10
+  version: 0.0.11
 platform: ruby
 authors:
 - M. C. McGrath
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-11-03 00:00:00.000000000 Z
+date: 2015-11-23 00:00:00.000000000 Z
 dependencies: []
 description: Crawls public LinkedIn profiles via Google
 email: shidash@shidash.com