indeedcrawler 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 0764adb113470fac8d62ee9eda673ae63d52d1cd
+  data.tar.gz: 6666aba0baff119ed76720feb85fa7a3cd16f9ca
+SHA512:
+  metadata.gz: 3032cd536c0a38062ebeaaa27e1470aa58319a2cd612d7428d62572d3498c8625ba744b80b70cbda7d769f5b0f06a0776c310dc178baa3f50edfc5d90c98b66b
+  data.tar.gz: da91a5282da3e4d55407edba9fe4c2c181d56b36b4c6640476893da531a440b8e32f8838232c9fc097ee4c47391286889cf39966809c9308c5565e2356ada1f0

data/lib/indeedcrawler.rb

@@ -0,0 +1,101 @@
+require 'json'
+require 'uri'
+require 'requestmanager'
+require 'nokogiri'
+require 'indeedparser'
+
+class IndeedCrawler
+  def initialize(search_query, location, proxy_list, wait_time, browser_num)
+    # Info for query
+    @search_query = search_query
+    @location = location
+
+    # Settings for request manager
+    @requests = RequestManager.new(proxy_list, wait_time, browser_num)
+
+    # Result tracking
+    @all_resume_links = Array.new
+    @output = Array.new
+  end
+
+  # Append query
+  def add_query(url)
+    url += "&q="+URI.encode_www_form([@search_query])
+  end
+
+  # Append location
+  def add_location(url)
+    url += "&" if @search_query
+    url += "l="+URI.encode_www_form([@location])
+  end
+
+  # Get the links on the page
+  def get_page_links(html)
+    # Get list of people
+    profiles = html.xpath("//li[@itemtype='http://schema.org/Person']")
+
+    # Get each profile link
+    profiles.each do |profile|
+      @all_resume_links.push("http://indeed.com/"+profile.xpath(".//a[@class='app_link']")[0]['href'])
+    end
+
+    # Navigate to next page if there's a class to do that
+    load_next_page(html) if !html.css("a.next").empty?
+  end
+
+  # Load the next page
+  def load_next_page(html)
+    next_html = load_restart_page("http://indeed.com/resumes"+html.css("a.next").first['href'], 0)
+    get_page_links(Nokogiri::HTML(next_html))
+  end
+
+  # Load the page and return or restart and retry if needed
+  def load_restart_page(url, count)
+    begin
+      return @requests.get_page(url)
+    rescue
+      if count < 2
+        @requests.restart_browser
+        load_restart_page(url, count+=1)
+      end
+    end
+  end
+
+  # Download and parse all resumes
+  def parse_resumes
+    @all_resume_links.each do |link|
+      resume = load_restart_page(link, 0)
+
+      begin
+        # Parse resume and add to results
+        i = IndeedParser.new(resume, link, {time_scraped: Time.now})
+        results = JSON.parse(i.get_results_by_job)
+      
+        results.each do |result|
+          @output.push(result)
+        end
+      rescue
+      end
+    end
+  end
+
+  # Get all the profile links
+  def collect_it_all
+    # Generate URL
+    url = "http://indeed.com/resumes?co=US"
+    url = add_query(url) if @search_query
+    url = add_location(url) if @location
+
+    # Get first page and navigate the rest
+    page_body = load_restart_page(url, 0)
+    html = Nokogiri::HTML(page_body)
+    get_page_links(html)
+
+    # Get and parse all results
+    parse_resumes
+
+    # Close browsers when done and return results
+    @requests.close_all_browsers
+    return JSON.pretty_generate(@output)
+  end
+end

metadata

@@ -0,0 +1,45 @@
+--- !ruby/object:Gem::Specification
+name: indeedcrawler
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- M. C. McGrath
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2015-12-23 00:00:00.000000000 Z
+dependencies: []
+description: Crawls Indeed resumes
+email: shidash@transparencytoolkit.org
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- lib/indeedcrawler.rb
+homepage: https://github.com/TransparencyToolkit/linkedincrawler
+licenses:
+- GPL
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.4.8
+signing_key: 
+specification_version: 4
+summary: Crawls Indeed resumes
+test_files: []
+has_rdoc: