linkedin-scraper 1.0.5 → 1.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 51ab67fc48225482a632be818fe7dc158c089cd2
-  data.tar.gz: 355c19cec2f5613b7dc8afd668e845dc406a3134
+  metadata.gz: 2c77bcfa337ca481575a21d58974d9f574c75578
+  data.tar.gz: e10ffd0759cb6535202a8977b9b9f3d0bb5530ce
 SHA512:
-  metadata.gz: 0fd8ef9fba5d09b4ceb466cffb94d6f8327eacd10cf18ced217715374c7931d9a1b7c8a4ddb94445f8652af71c31e07ae862184acba37f0bfe8209ae1e9b7a52
-  data.tar.gz: fa66286cb67770d52151d190e2f4b23ab3a3ebcc80d3201d6c87e3bf3b39ca98ca74254a4041e612a59c4b3d50a5ddcae90d5eaba9da53017da4116efe7224af
+  metadata.gz: 177977da49150f249c6bc7f5d7dcbaaf4ac2d167c8e4835c8e1d799e5ff126704972d959b4acd86db0b1a93baa0c43233621367721ced19bcee521c39ca317ea
+  data.tar.gz: c1e314563caecc20c601abcf1769846a20b807b0966dbfc804a152c721d0f116c9ab5156a9a5aceb0d6c76799281da7745eb24543979e9205a41508f9f68d03e

data/README.md

@@ -18,7 +18,9 @@ Install the gem from RubyGems:
 This gem is tested on 1.9.2, 1.9.3, 2.0.0, 2.2, 2.3, JRuby1.9, rbx1.9,
 
 ## Usage
+Include the gem
 
+    require 'linkedin_scraper'
 
 Initialize a scraper instance
 
@@ -270,7 +272,7 @@ It takes the url as the first argument.
 
 Bug reports and pull requests are welcome on GitHub at https://github.com/yatish27/linkedin-scraper.
 This project is intended to be a safe, welcoming space for collaboration, and contributors are expected to adhere to the
-[Contributor Covenant](contributor-covenant.org) code of conduct.
+[Contributor Covenant](http://contributor-covenant.org) code of conduct.
 
 
 ## License

data/lib/linkedin_scraper.rb

@@ -2,4 +2,5 @@ require "rubygems"
 require "mechanize"
 require "cgi"
 require "net/http"
+require "random_user_agent"
 Dir["#{File.expand_path(File.dirname(__FILE__))}/linkedin_scraper/*.rb"].each { |file| require file }

data/lib/linkedin_scraper/profile.rb

@@ -2,25 +2,6 @@
 module Linkedin
   class Profile
 
-    #USER_AGENTS = ["Windows IE 6", "Windows IE 7", "Windows Mozilla", "Mac Safari", "Mac Firefox", "Mac Mozilla", "Linux Mozilla", "Linux Firefox", "Linux Konqueror"]
-    USER_AGENTS = [
-      "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.8.1.6) Gecko/20070725 Firefox/2.0.0.6",
-      "Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:5.0) Gecko/20100101 Firefox/5.0",
-      "Mozilla/5.0 (Windows NT 6.1.1; rv:5.0) Gecko/20100101 Firefox/5.0",
-      "Mozilla/5.0 (X11; U; Linux i586; de; rv:5.0) Gecko/20100101 Firefox/5.0",
-      "Mozilla/5.0 (X11; Linux i686) AppleWebKit/535.1 (KHTML, like Gecko) Ubuntu/11.04 Chromium/14.0.825.0 Chrome/14.0.825.0 Safari/535.1",
-      "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/535.1 (KHTML, like Gecko) Chrome/14.0.824.0 Safari/535.1",
-      "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.7; rv:5.0) Gecko/20100101 Firefox/5.0",
-      "Mozilla/5.0 (Macintosh; PPC MacOS X; rv:5.0) Gecko/20110615 Firefox/5.0",
-      "Mozilla/5.0 (Windows; U; MSIE 9.0; WIndows NT 9.0; en-US))",
-      "Mozilla/5.0 (compatible; MSIE 8.0; Windows NT 5.2; Trident/4.0; Media Center PC 4.0; SLCC1; .NET CLR 3.0.04320)",
-      "Mozilla/5.0 (Windows; U; MSIE 7.0; Windows NT 6.0; en-US)",
-      "Mozilla/5.0 (compatible; Konqueror/4.5; FreeBSD) KHTML/4.5.4 (like Gecko)",
-      "Opera/9.80 (Windows NT 6.1; U; es-ES) Presto/2.9.181 Version/12.00",
-      "Opera/9.80 (X11; Linux x86_64; U; fr) Presto/2.9.168 Version/11.50",
-      "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_8; de-at) AppleWebKit/533.21.1 (KHTML, like Gecko) Version/5.0.5 Safari/533.21.1",
-      "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_7; da-dk) AppleWebKit/533.21.1 (KHTML, like Gecko) Version/5.0.5 Safari/533.21.1"
-    ]
     ATTRIBUTES = %w(
     name
     first_name
@@ -101,7 +82,7 @@ module Linkedin
     end
 
     def skills
-      @skills ||= (@page.search(".pills .skill").map { |skill| skill.text.strip if skill.text } rescue nil)
+      @skills ||= (@page.search(".pills .skill:not(.see-less)").map { |skill| skill.text.strip if skill.text } rescue nil)
     end
 
     def past_companies
@@ -185,13 +166,13 @@ module Linkedin
     def projects
       @projects ||= @page.search("#projects .project").map do |project|
         p = {}
-        start_date, end_date = project.at("date-range").text.gsub(/\s+|\n/, " ").strip.split(" – ") rescue nil
+        start_date, end_date = project.at(".date-range").text.gsub(/\s+|\n/, " ").strip.split(" – ") rescue nil
 
         p[:title] = project.at(".item-title").text
         p[:link] =  CGI.parse(URI.parse(project.at(".item-title a")['href']).query)["url"][0] rescue nil
         p[:start_date] = parse_date(start_date) rescue nil
         p[:end_date] = parse_date(end_date)  rescue nil
-        p[:description] = project.at(".description").text rescue nil
+        p[:description] = project.at(".description").children().to_s rescue nil
         p[:associates] = project.search(".contributors .contributor").map{ |c| c.at("a").text } rescue nil
         p
       end
@@ -215,10 +196,12 @@ module Linkedin
         company = {}
         company[:title] = node.at(".item-title").text.gsub(/\s+|\n/, " ").strip if node.at(".item-title")
         company[:company] = node.at(".item-subtitle").text.gsub(/\s+|\n/, " ").strip if node.at(".item-subtitle")
+        company[:location] = node.at(".location").text if node.at(".location")
         company[:description] = node.at(".description").text.gsub(/\s+|\n/, " ").strip if node.at(".description")
+        company[:company_logo] = node.at(".logo a img").first[1] if node.at(".logo")
 
-        start_date, end_date = node.at(".meta").text.strip.split(" – ") rescue nil
-        company[:duration] = node.at(".meta").text[/.*\((.*)\)/, 1]
+        start_date, end_date = node.at(".date-range").text.strip.split(" – ") rescue nil
+        company[:duration] = node.at(".date-range").text[/.*\((.*)\)/, 1]
         company[:start_date] = parse_date(start_date) rescue nil
 
         if end_date && end_date.match(/Present/)
@@ -261,7 +244,7 @@ module Linkedin
 
     def http_client
       Mechanize.new do |agent|
-        agent.user_agent = USER_AGENTS.sample
+        agent.user_agent = RandomUserAgent.randomize
         unless @options.empty?
           agent.set_proxy(@options[:proxy_ip], @options[:proxy_port], @options[:username], @options[:password])
         end

data/lib/linkedin_scraper/version.rb

@@ -1,5 +1,5 @@
 module Linkedin
   module Scraper
-    VERSION = '1.0.5'
+    VERSION = '1.1.0'
   end
 end

data/linkedin-scraper.gemspec

@@ -16,7 +16,8 @@ Gem::Specification.new do |gem|
   gem.license       = "MIT"
 
   gem.add_dependency 'mechanize', '~> 2'
-  
+  gem.add_dependency 'random_user_agent'
+
   gem.add_development_dependency 'rspec', '~> 3'
   gem.add_development_dependency 'rake', '~> 10'
 end

data/spec/linkedin_scraper/profile_spec.rb

@@ -47,6 +47,10 @@ describe Linkedin::Profile do
     it "returns list of profile's skills" do
       expect(profile.skills).to include("Product Development")
     end
+
+    it 'does not return "See less"' do
+      expect(profile.skills).not_to include("See less")
+    end
   end
 
   describe '#websites' do

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: linkedin-scraper
 version: !ruby/object:Gem::Version
-  version: 1.0.5
+  version: 1.1.0
 platform: ruby
 authors:
 - Yatish Mehta
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-03-09 00:00:00.000000000 Z
+date: 2016-05-06 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: mechanize
@@ -24,6 +24,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '2'
+- !ruby/object:Gem::Dependency
+  name: random_user_agent
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement