linkedincrawler 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 266cfd3d2297b67906c17aa1781ddf37b9519683
+  data.tar.gz: 9c46996cd2eef74e646e4378099afcfe5d357cc7
+SHA512:
+  metadata.gz: 350fa16c241578c88e840b9ccaab17267bdc9b7af9bb38882e6b73a5a9c72b06511b752dc8e47015bc2698840b38870559c5bd40f58a4d3cdb914884244cc6cd
+  data.tar.gz: de3aef7908c810900828f4a9a399cf939950ddf32495835dba4cd613804b80044e4f9757f5cec707b2f009aa680ca2fbf5eb0c545e046e08f323c3603822c6d2

data/lib/linkedin_crawler.rb

@@ -0,0 +1,47 @@
+require 'linkedinparser'
+require 'generalscraper'
+require 'selenium-webdriver'
+require 'pry'
+
+class LinkedinCrawler
+  include ProxyManager
+  def initialize(search_terms)
+    @search_terms = search_terms
+    @output = Array.new
+  end
+
+  # Run search terms and get results
+  def search
+    # Run Google search
+    g = GeneralScraper.new("site:linkedin.com/pub", @search_terms,  "/home/shidash/proxies", false)
+    
+    # Scrape each resulting LinkedIn page
+    gen_driver
+    JSON.parse(g.getURLs).each do |profile|
+      scrape(profile)
+    end
+  end
+
+  # Generate driver for searches
+  def gen_driver
+    profile = Selenium::WebDriver::Firefox::Profile.new
+    profile['intl.accept_languages'] = 'en'
+    profile["javascript.enabled"] = false
+    @driver = Selenium::WebDriver.for :firefox, profile: profile
+  end
+
+  # Scrape each page
+  def scrape(profile_url)
+    # Get profile page
+    profile_html = getPage(profile_url, @driver, nil, 5, false).page_source
+
+    # Parse profile and add to output
+    l = LinkedinParser.new(profile_html, profile_url, {timestamp: Time.now})
+    @output += JSON.parse(l.results_by_job)
+  end
+
+  # Print output in JSON
+  def gen_json
+    JSON.pretty_generate(@output)
+  end
+end

metadata

@@ -0,0 +1,87 @@
+--- !ruby/object:Gem::Specification
+name: linkedincrawler
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- M. C. McGrath
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2015-11-01 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: linkedinparser
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: generalscraper
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: selenium-webdriver
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Crawls public LinkedIn profiles via Google
+email: shidash@shidash.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- lib/linkedin_crawler.rb
+homepage: https://github.com/TransparencyToolkit/linkedincrawler
+licenses:
+- GPL
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.4.8
+signing_key: 
+specification_version: 4
+summary: Crawls public LinkedIn profiles
+test_files: []
+has_rdoc: