linkedincrawler 0.0.5 → 0.0.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: c1933a4a905a2b52b2d3b888d54c4d8d044ec7c5
-  data.tar.gz: 771175b55c592fc4bf14317a1c63cc76efb4ecd9
+  metadata.gz: 99433021962017d0dc2ef40342127cce06abac54
+  data.tar.gz: 4a0a0a5441e6a9bcdb76ce69b9fb1be8f3647619
 SHA512:
-  metadata.gz: a427d86d2ddc648e1bb6362b93f97ffb56e1f68420ce418d6b4754062e752b8961b8e4e72da6a465c990d69367097e37e1d3a9ac054394e077de6449ca1fe62d
-  data.tar.gz: 23618949f1ce1bd4eaf16589bed5078e6cc840b36f8c8624f6e477e0f89890c48ac1c5efa5a8d2d191119689509f5284090714c8380693d1ae7fc93bd014c92f
+  metadata.gz: ec59a52e5c73cdf9f3d91bd4e7936a2957c7671ee92d5566f06ba64ae230380ecfcb00aa52093bed3c84b5909309f1b1f41d3b40d78968b66a116c8215f149ba
+  data.tar.gz: 1bafd3d56a9b3ad9422a25ef2118cb610aaa229eb899c63a070fa46c8c933b65052402f8191246421fbaa8983910591a3fbb38bce704d48f0c6e1457a8b7f549

data/lib/linkedincrawler.rb

@@ -19,11 +19,21 @@ class LinkedinCrawler
   def search
     # Run Google search
     g = GeneralScraper.new("site:linkedin.com/pub -site:linkedin.com/pub/dir/", @search_terms, @proxy_list)
-    
-    # Scrape each resulting LinkedIn page
-    JSON.parse(g.getURLs).each do |profile|
-      if profile.include?(".linkedin.") && !profile.include?("/search")
-        scrape(profile)
+    begin
+      urls = g.getURLs
+    rescue # Search again if it didn't work the first time
+      search
+    end
+
+    # Search again if it didn't run
+    if urls.length == 0 || urls.empty?
+      search
+    else
+      # Scrape each resulting LinkedIn page
+      JSON.parse(urls).each do |profile|
+        if profile.include?(".linkedin.") && !profile.include?("/search")
+          scrape(profile)
+        end
       end
     end
 
@@ -64,3 +74,6 @@ class LinkedinCrawler
     JSON.pretty_generate(@output)
   end
 end
+c = LinkedinCrawler.new("remotely piloted aircraft", 5, "/home/shidash/proxylist2", [3, 5])
+c.search
+File.write("test.json", c.gen_json)

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: linkedincrawler
 version: !ruby/object:Gem::Version
-  version: 0.0.5
+  version: 0.0.6
 platform: ruby
 authors:
 - M. C. McGrath