linkedin-scraper 0.0.6 → 0.0.7

data/README.rdoc

@@ -32,34 +32,87 @@ Then you can see the scraped data like this:
   profile.industry            #the domain for which the contact belongs
     
   profile.past_companies
-  #Array of hash containing its past job companies and job profile
-  #Example
-  #  [
-  #    [0] {
-  #          :past_title => "Intern",
-  #        :past_company => "Sungard"
-  #        },
-  #    [1] {
-  #          :past_title => "Software Developer",
-  #        :past_company => "Microsoft"
-  #        }
-  #  ]
+    #Array of hash containing its past job companies and job profile
+    #Example
+    #  [
+    #     [0] {
+    #                 :past_company => "Consumyze Software",
+    #                   :past_title => "Trainee",
+    #         :past_company_website => "http://www.consumyze.com",
+    #                  :description => "Responsible for design and development"
+    #     },
+    #     [1] {
+    #                 :past_company => "SunGard Global Services",
+    #                   :past_title => "Project Intern",
+    #         :past_company_website => "http://www.sungard.com/globalservices/learnmore",
+    #                  :description => "Fame PassPoint. Developed an entirely Ajax based online control panel for user management and Data access for Fame"
+    #     }
+    # ]
   profile.current_companies
   #Array of hash containing its current job companies and job profile
-  #Example
-  #  [
-  #    [0] {
-  #          :current_title => "Intern",
-  #        :current_company => "Sungard"
-  #        },
-  #    [1] {
-  #          :current_title => "Software Developer",
-  #        :current_company => "Microsoft"
-  #        }
-  #  ]
+    #Example
+    #  [
+    #    [0] {
+    #          :current_title => "Intern",
+    #        :current_company => "Sungard"
+    #     :current_company_url=>"http://www.betterlabs.net",
+    #             :description=>"Responsible for design and development of projects on Ruby on Rails."
+    #        },
+    #    [1] {
+    #          :current_title  => "Software Developer",
+    #        :current_company  => "Microsoft"
+    #     :current_company_url =>"http://www.microsoft.net",
+    #             :description =>"Development and design"
+
+    #        }
+    #  ]
     
     
   profile.linkedin_url        #url of the profile
+
+  profile.websites
+  #Array of websites 
+  #[
+  # [0] "http://www.yatishmehta.in"
+  #]
+
+  profile.groups
+    #array of hashes containing group name and link
+    # [
+    #     [ 0] {
+    #         :name => "Business on Rails",
+    #         :link => "http://www.linkedin.com/groups/Business-on-Rails-27822"
+    #     },
+    #     [ 1] {
+    #         :name => "HTML5 Technologies",
+    #         :link => "http://www.linkedin.com/groups/HTML5-Technologies-2868882"
+    #     },
+    #     [ 2] {
+    #         :name => "India on Rails",
+    #         :link => "http://www.linkedin.com/groups/India-on-Rails-149940"
+    #         :name => "Open Source",
+    #         :link => "http://www.linkedin.com/groups?gid=43875"
+    #     },
+    #     [ 4] {
+    #         :name => "Rails Developers",
+    #         :link => "http://www.linkedin.com/groups?gid=77764"
+    #     },
+    # ]
+
+  profile.education
+  #Array of hashes for eduction
+    # [
+    #     [0] {
+    #                :name => "Vishwakarma Institute of Technology",
+    #         :description => "B.Tech, Computer Engineering",
+    #              :period => "2007 – 2011"
+    #     },
+    #     [1] {
+    #                :name => "St Ursula's High School",
+    #         :description => "Secondary School Education",
+    #              :period => nil
+    #     }
+    # ]
     
   profile.recommended_visitors
   #Its the list of visitors "Viewers of this profile also viewed..."

data/lib/linkedin-scraper.rb

@@ -1,6 +1,7 @@
-require "linkedin-scraper/version"
 require "rubygems"
 require "mechanize"
+require "cgi"
+require "net/http"
 Dir["#{File.expand_path(File.dirname(__FILE__))}/linkedin-scraper/*.rb"].each {|file| require file }
 
 

data/lib/linkedin-scraper/profile.rb

@@ -3,22 +3,68 @@ module Linkedin
   class Profile    
     #the First name of the contact
     attr_accessor :first_name,:last_name,:title,:location,:country,
-                  :industry, :linkedin_url,:recommended_visitors,:profile,
-                  :page
+    :industry, :linkedin_url,:recommended_visitors,:page
+    #Array of hashes for eduction
+    # [
+    #     [0] {
+    #                :name => "Vishwakarma Institute of Technology",
+    #         :description => "B.Tech, Computer Engineering",
+    #              :period => "2007 – 2011"
+    #     },
+    #     [1] {
+    #                :name => "St Ursula's High School",
+    #         :description => "Secondary School Education",
+    #              :period => nil
+    #     }
+    # ]
+    attr_accessor :education
+
+    #Array of websites 
+    #[
+    #[0] "http://www.yatishmehta.in"
+    #]
+    attr_accessor :websites
+    #array of hashes containing group name and link
+    # [
+    #     [ 0] {
+    #         :name => "Business on Rails",
+    #         :link => "http://www.linkedin.com/groups/Business-on-Rails-27822"
+    #     },
+    #     [ 1] {
+    #         :name => "HTML5 Technologies",
+    #         :link => "http://www.linkedin.com/groups/HTML5-Technologies-2868882"
+    #     },
+    #     [ 2] {
+    #         :name => "India on Rails",
+    #         :link => "http://www.linkedin.com/groups/India-on-Rails-149940"
+    #         :name => "Open Source",
+    #         :link => "http://www.linkedin.com/groups?gid=43875"
+    #     },
+    #     [ 4] {
+    #         :name => "Rails Developers",
+    #         :link => "http://www.linkedin.com/groups?gid=77764"
+    #     },
+    # ]
+    attr_accessor:groups
 
 
     #Array of hash containing its past job companies and job profile
     #Example
     #  [
-    #    [0] {
-    #          :past_title => "Intern",
-    #        :past_company => "Sungard"
-    #        },
-    #    [1] {
-    #          :past_title => "Software Developer",
-    #        :past_company => "Microsoft"
-    #        }
-    #  ]
+    #     [0] {
+    #                 :past_company => "Consumyze Software",
+    #                   :past_title => "Trainee",
+    #         :past_company_website => "http://www.consumyze.com",
+    #                  :description => "Responsible for design and development"
+    #     },
+    #     [1] {
+    #                 :past_company => "SunGard Global Services",
+    #                   :past_title => "Project Intern",
+    #         :past_company_website => "http://www.sungard.com/globalservices/learnmore",
+    #                  :description => "Fame PassPoint. Developed an entirely Ajax based online control panel for user management and Data access for Fame"
+    #     }
+    # ]
+
 
     attr_accessor :past_companies
     #Array of hash containing its current job companies and job profile
@@ -27,10 +73,15 @@ module Linkedin
     #    [0] {
     #          :current_title => "Intern",
     #        :current_company => "Sungard"
+    #     :current_company_url=>"http://www.betterlabs.net",
+    #             :description=>"Responsible for design and development of projects on Ruby on Rails."
     #        },
     #    [1] {
-    #          :current_title => "Software Developer",
-    #        :current_company => "Microsoft"
+    #          :current_title  => "Software Developer",
+    #        :current_company  => "Microsoft"
+    #     :current_company_url =>"http://www.microsoft.net",
+    #             :description =>"Development and design"
+
     #        }
     #  ]
     attr_accessor :current_companies
@@ -47,10 +98,14 @@ module Linkedin
       @current_companies=get_current_companies page
       @past_companies=get_past_companies page
       @recommended_visitors=get_recommended_visitors page
+      @education=get_education page
       @linkedin_url=url
+      @websites=get_websites page
+      @groups=get_groups page
       @page=page
     end
     #returns:nil if it gives a 404 request
+
     def self.get_profile url
       begin
         @agent=Mechanize.new
@@ -63,6 +118,17 @@ module Linkedin
       end
     end
 
+    def get_company_url node
+      if node.at("h4/strong/a")
+        link=node.at("h4/strong/a")["href"]
+        @agent=Mechanize.new
+        @agent.user_agent_alias = USER_AGENTS.sample
+        @agent.max_history = 0
+        page=@agent.get("http://www.linkedin.com"+link)
+        url=page.at(".basic-info/div/dl/dd/a").text if page.at(".basic-info/div/dl/dd/a")
+      end
+    end
+
     private
 
     def get_first_name page
@@ -91,12 +157,13 @@ module Linkedin
 
     def get_past_companies page
       past_cs=[]
-      if page.search(".past").first
-        page.search(".past").search("li").each do |past_company|
-          title,company=past_company.text.strip.split(" at ")
-          company=company.gsub(/\s+/, " ").strip if company
-          title=title.gsub(/\s+/, " ").strip if title
-          past_company={:past_company=>company,:past_title=> title}
+      if page.search(".position.experience.vevent.vcard.summary-past").first
+        page.search(".position.experience.vevent.vcard.summary-past").each do |past_company|
+          url=get_company_url past_company
+          title=past_company.at("h3").text.gsub(/\s+|\n/, " ").strip if past_company.at("h3")
+          company=past_company.at("h4").text.gsub(/\s+|\n/, " ").strip if past_company.at("h4")
+          description=past_company.at(".description.past-position").text.gsub(/\s+|\n/, " ").strip if past_company.at(".description.past-position")
+          past_company={:past_company=>company,:past_title=> title,:past_company_website=>url,:description=>description}
           past_cs<<past_company
         end
         return past_cs
@@ -105,18 +172,62 @@ module Linkedin
 
     def get_current_companies page
       current_cs=[]
-      if page.search(".current").first
-        page.search(".current").search("li").each do |past_company|
-          title,company=past_company.text.strip.split(" at ")
-          company=company.gsub(/\s+/, " ").strip if company
-          title=title.gsub(/\s+/, " ").strip if title
-          current_company={:current_company=>company,:current_title=> title}
+      if page.search(".position.experience.vevent.vcard.summary-current").first
+        page.search(".position.experience.vevent.vcard.summary-current").each do |current_company|
+          url=get_company_url current_company
+          title=current_company.at("h3").text.gsub(/\s+|\n/, " ").strip if current_company.at("h3")
+          company=current_company.at("h4").text.gsub(/\s+|\n/, " ").strip if current_company.at("h4")
+          description=current_company.at(".description.current-position").text.gsub(/\s+|\n/, " ").strip if current_company.at(".description.current-position")
+          current_company={:current_company=>company,:current_title=> title,:current_company_url=>url,:description=>description}
           current_cs<<current_company
         end
         return current_cs
       end
     end
 
+    def get_education page
+      education=[]
+      if page.search(".position.education.vevent.vcard").first
+        page.search(".position.education.vevent.vcard").each do |item|
+          name=item.at("h3").text.gsub(/\s+|\n/, " ").strip if item.at("h3")
+          desc=item.at("h4").text.gsub(/\s+|\n/, " ").strip if item.at("h4")
+          period=item.at(".period").text.gsub(/\s+|\n/, " ").strip if item.at(".period")
+          edu={:name=>name,:description=>desc,:period=>period} 
+          education<<edu
+        end
+        return education
+      end
+    end
+
+    def get_websites page
+      websites=[]
+      if page.search(".website").first
+        page.search(".website").each do |site|
+          url=site.at("a")["href"]
+          url="http://www.linkedin.com"+url
+          url=CGI.parse(URI.parse(url).query)["url"]
+          websites<<url
+        end
+        return websites.flatten!
+      end  
+    end
+
+    def get_groups page
+      groups=[]
+      if page.search(".group-data").first
+        page.search(".group-data").each do |item|
+          name=item.text.gsub(/\s+|\n/, " ").strip
+          link="http://www.linkedin.com"+item.at("a")["href"]
+          groups<<{:name=>name,:link=>link}
+        end
+        return groups
+      end
+
+    end
+
+
+
+
     def get_recommended_visitors  page
       recommended_vs=[]
       if page.search(".browsemap").first
@@ -130,7 +241,6 @@ module Linkedin
         end
         return recommended_vs
       end
-      
     end
   end
 end

data/lib/linkedin-scraper/version.rb

@@ -1,5 +1,5 @@
 module Linkedin
   module Scraper
-    VERSION = "0.0.6"
+    VERSION = "0.0.7"
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: linkedin-scraper
 version: !ruby/object:Gem::Version
-  version: 0.0.6
+  version: 0.0.7
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-07-23 00:00:00.000000000 Z
+date: 2012-08-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: mechanize