linkedincrawler 0.0.4 → 0.0.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 3be949ebe25839f19c8c03dc0be404dc8e15928f
-  data.tar.gz: b070cd1c81b4e91d31f38d13856279c27c9455ff
+  metadata.gz: c1933a4a905a2b52b2d3b888d54c4d8d044ec7c5
+  data.tar.gz: 771175b55c592fc4bf14317a1c63cc76efb4ecd9
 SHA512:
-  metadata.gz: fae6bb4559925406d27e631e08c901d5d86a7089b5d94f7e31ebe16f6849d284da8a4070af5d3931de73ea17a1cb3779eb728a8b9036989dfb5cdc6873588727
-  data.tar.gz: f100d18fd9951c0baa93e5dbbe6be08a4977cfb4aade1cb2e872d6bfecb6bc5f7f082ce85b4a5e671a1507287f9cd6cf7d1ab84121d36cd9c02be54984197646
+  metadata.gz: a427d86d2ddc648e1bb6362b93f97ffb56e1f68420ce418d6b4754062e752b8961b8e4e72da6a465c990d69367097e37e1d3a9ac054394e077de6449ca1fe62d
+  data.tar.gz: 23618949f1ce1bd4eaf16589bed5078e6cc840b36f8c8624f6e477e0f89890c48ac1c5efa5a8d2d191119689509f5284090714c8380693d1ae7fc93bd014c92f

data/lib/linkedincrawler.rb

@@ -1,45 +1,61 @@
+require 'requestmanager'
 require 'linkedinparser'
 require 'generalscraper'
+
 require 'selenium-webdriver'
 require 'pry'
 
 class LinkedinCrawler
-  include ProxyManager
-  def initialize(search_terms)
+  def initialize(search_terms, retry_limit, proxy_list, request_time)
     @search_terms = search_terms
     @output = Array.new
+    @retry_limit = retry_limit
+    @retry_count = 0
+    @proxy_list = proxy_list
+    @requests = RequestManager.new(@proxy_list, request_time, 5)
   end
 
   # Run search terms and get results
   def search
     # Run Google search
-    g = GeneralScraper.new("site:linkedin.com/pub", @search_terms,  "/home/shidash/proxies", false)
+    g = GeneralScraper.new("site:linkedin.com/pub -site:linkedin.com/pub/dir/", @search_terms, @proxy_list)
     
     # Scrape each resulting LinkedIn page
-    gen_driver
     JSON.parse(g.getURLs).each do |profile|
-      scrape(profile)
+      if profile.include?(".linkedin.") && !profile.include?("/search")
+        scrape(profile)
+      end
     end
+
+    # Close all the browsers
+    @requests.close_all_browsers
   end
 
-  # Generate driver for searches
-  def gen_driver
-    profile = Selenium::WebDriver::Firefox::Profile.new
-    profile['intl.accept_languages'] = 'en'
-    profile["javascript.enabled"] = false
-    @driver = Selenium::WebDriver.for :firefox, profile: profile
+  # Check that it is actually a LinkedIn profile page
+  def check_right_page(profile_url)
+    return !profile_url.include?("www.google") &&
+           !profile_url.include?("linkedin.com/pub/dir") &&
+           !profile_url.include?("/search") &&
+           @retry_count < @retry_limit
   end
 
   # Scrape each page
   def scrape(profile_url)
     # Get profile page
-    profile_html = getPage(profile_url, @driver, nil, 5, false).page_source
+    profile_html = @requests.get_page(profile_url)
 
     # Parse profile and add to output
     begin
       l = LinkedinParser.new(profile_html, profile_url, {timestamp: Time.now})
       @output += JSON.parse(l.results_by_job)
+      @retry_count = 0
     rescue
+      # If proxy doesn't work, try another a few times
+      if check_right_page(profile_url)
+        @requests.restart_browser
+        @retry_count += 1
+        scrape(profile_url)
+      end
     end
   end
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: linkedincrawler
 version: !ruby/object:Gem::Version
-  version: 0.0.4
+  version: 0.0.5
 platform: ruby
 authors:
 - M. C. McGrath
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-11-01 00:00:00.000000000 Z
+date: 2015-11-03 00:00:00.000000000 Z
 dependencies: []
 description: Crawls public LinkedIn profiles via Google
 email: shidash@shidash.com