tsjobcrawler 0.1

data/Gemfile

@@ -0,0 +1,7 @@
+source 'https://rubygems.org'
+gemspec
+
+group :test do
+  gem "simplecov"
+  gem "codeclimate-test-reporter", "~> 1.0.0"
+end

data/README.md

@@ -0,0 +1,28 @@
+This is a crawler for job listings that require security clearance.
+
+To run-
+
+t = TSJobCrawler.new("search term" (or nil), request_manager, cm_hash or nil)
+
+t.crawl_jobs
+
+
+For example-
+
+Headless.ly do
+
+  r = RequestManager.new(nil, [0, 0], 1)
+  
+  t = TSJobCrawler.new(nil, r, nil)
+  
+  t.crawl_jobs
+  
+  File.write("test.json", t.gen_json)
+  
+end
+
+
+If you input nil for the search term, it downloads as many job listings as
+possible. Unless you have a lot of RAM, you should run it through Harvester if
+you want to download as many listings as possible as then you can take
+advantage of incremental result reporting.

data/bin/console

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+
+require "bundler/setup"
+require "tsjobcrawler"
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+# require "pry"
+# Pry.start
+
+require "irb"
+IRB.start

data/bin/setup

@@ -0,0 +1,7 @@
+#!/bin/bash
+set -euo pipefail
+IFS=$'\n\t'
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/lib/clearancejobscom/clearance_jobs_com_crawler.rb

@@ -0,0 +1,86 @@
+require 'pry'
+require 'open-uri'
+require 'nokogiri'
+require 'cgi'
+require 'json'
+require 'requestmanager'
+require 'headless'
+require 'harvesterreporter'
+
+load 'clearancejobscom/clearance_jobs_com_parser.rb'
+load 'util/failure_handler.rb'
+
+class ClearanceJobsComCrawler
+  include FailureHandler
+  def initialize(search_term, requests=nil, cm_hash=nil)
+    @search_term = search_term
+    @requests = requests
+    @base_url = set_base_url
+
+    # Handle crawler manager info
+    @reporter = HarvesterReporter.new(cm_hash)
+  end
+
+  # Run the crawler
+  def crawl
+    page_count = get_page_count
+
+    (1..page_count).each do |page_num|
+      listing_links = collect_links_on_page(get_next_page(page_num))
+      parse_listings(listing_links)
+    end
+  end
+
+  # Get base url
+  def set_base_url
+    if @search_term == nil
+      @base_url = "https://www.clearancejobs.com/jobs?"
+    else
+      @base_url = "https://www.clearancejobs.com/jobs?keywords="+CGI.escape(@search_term)+"&zip_text="
+    end
+  end
+
+  # Get the URL for the next page
+  def get_next_page_url(page_num)
+    return @base_url+"PAGE="+page_num.to_s+"&limit=25"
+  end
+
+  # Get the page
+  def get_page(url)
+    get_retry(url, @requests, 0)
+  end
+
+  # Get the correct total # of pages
+  def get_page_count
+    page_html = Nokogiri::HTML.parse(get_next_page(1))
+    result_count = page_html.css("#viewing").text.split(" of ")[1].gsub(",", "").to_i
+    return (result_count/25.0).ceil
+  end
+
+  # Get the next page
+  def get_next_page(page_num)
+    return get_page(get_next_page_url(page_num))
+  end
+
+  # Collect the links on the page
+  def collect_links_on_page(page)
+    html = Nokogiri::HTML.parse(page)
+    return html.css(".cj-search-result-item-title").css("a").map{|a| a['href']}
+  end
+
+  # Parse the listings on the page
+  def parse_listings(listings)
+    found_listings = Array.new
+    listings.each do |listing|
+      parser = ClearanceJobsComParser.new(listing, get_page(listing), @requests)
+      parsed_listing = parser.parse
+      found_listings.push(parsed_listing) if parsed_listing
+    end
+
+    @reporter.report_results(found_listings, listings.first)
+  end
+
+  def gen_json
+    return @reporter.gen_json
+  end
+end

data/lib/clearancejobscom/clearance_jobs_com_parser.rb

@@ -0,0 +1,137 @@
+require 'pry'
+require 'open-uri'
+require 'nokogiri'
+
+load 'util/failure_handler.rb'
+
+class ClearanceJobsComParser
+  include FailureHandler
+  def initialize(url, page, requests=nil)
+    @url = url
+    @requests = requests
+    @i = 0
+    @html = page
+    @page = Nokogiri::HTML.parse(page)
+  end
+
+  # Parse the profile
+  def parse
+    begin
+      return {
+        url: @url,
+        company_name: company_name,
+        location: location,
+        job_title: job_title,
+        job_description: job_description,
+        job_description_plaintext: job_description_plaintext,
+        required_travel: required_travel,
+        salary: salary,
+        salary_notes: salary_notes,
+        job_category: job_category,
+        group_id: group_id,
+        required_experience: required_experience,
+        employment_status: employment_status,
+        required_clearance: required_clearance,
+        required_experience: required_experience,
+        work_environment: work_environment,
+        posting_date: posting_date,
+        html: @html
+      }
+    rescue
+      @i += 1
+      if @i < 10
+        @html = Nokogiri::HTML.parse(get_retry(@url, @requests, @i))
+        parse
+      end
+    end
+  end
+  
+  # Get the company name
+  def company_name
+    @page.css("h2").text
+  end
+
+  # Get the job location
+  def location
+    raw_location = @page.css("div").select{|e| e['itemprop'] == "hiringOrganization"}[0].css("h3").text
+    raw_location.gsub(/(\d)/, "").strip if raw_location
+  end
+
+  # Get the job title
+  def job_title
+    @page.css("h1").text
+  end
+
+  # Get the job description
+  def job_description
+    @page.css("div.margin-bottom-20").select{|e| e['itemprop'] == "description"}[0].to_html
+  end
+
+  # Get the job description without text
+  def job_description_plaintext
+    Nokogiri::HTML.parse(job_description.gsub('<br />',"\n").gsub('<br>', "\n").gsub('<br/>', "\n")).text
+  end
+  
+  # Get if there is travel required
+  def required_travel
+    get_element_value("Travel:")
+  end
+  
+  # Get the salary
+  def salary
+    get_element_value("Compensation:")
+  end
+
+  # Get notes about the salary
+  def salary_notes
+    salary_info = get_element_value("Compensation Comments:")
+    salary_info.lstrip.strip if salary_info
+  end
+
+  # Get the job category
+  def job_category
+    get_element_value("Job Category:")
+  end
+
+  # Get the group ID
+  def group_id
+    get_element_value("Group ID")
+  end
+
+  # Get the # of years of experience required
+  def required_experience
+    get_element_value("Minimum Experience Required")  
+  end
+
+  # Get the employment status for the position
+  def employment_status
+    get_element_value("Status:")
+  end
+
+  # Get the clearance level
+  def required_clearance
+    get_element_value("Minimum Clearance")
+  end
+
+  # Get the work environment
+  def work_environment
+    get_element_value("Workplace:")
+  end
+
+  # Get the date of the posting
+  def posting_date
+    element = get_element("Post Date:")
+    DateTime.parse(element[0].css("meta")[0]['content'])
+  end
+
+  # Get the value for the element
+  def get_element_value(phrase)
+    element = get_element(phrase)[0]
+    element.css("strong").text if element
+  end
+
+  # Get the element including the phrase specified
+  def get_element(phrase)
+    @page.css(".cj-job-data").select{|d| d.text.include?(phrase) }
+  end
+end

data/lib/clearedjobsnet/cleared_jobs_net_crawler.rb

@@ -0,0 +1,141 @@
+require 'nokogiri'
+require 'open-uri'
+require 'cgi'
+require 'json'
+require 'requestmanager'
+require 'harvesterreporter'
+
+load 'clearedjobsnet/cleared_jobs_net_parser.rb'
+load 'util/failure_handler.rb'
+
+class ClearedJobsNetCrawler
+  include FailureHandler
+  def initialize(crawl_type, filter_name=nil, requests=nil, cm_hash=nil)
+    @base_url = "https://clearedjobs.net/"
+    @output = Array.new
+    @requests = requests
+
+    # Handle crawler manager info
+    @reporter = HarvesterReporter.new(cm_hash)
+    
+    # Get all items
+    if crawl_type == "all"
+      @crawl_type = crawl_type
+
+    # Get a company page
+    elsif filter_name == "company_page"
+      @crawl_type = "company_page"
+      @search_term = crawl_type
+
+    # Add a filter
+    elsif filter_name
+      @crawl_type = "filter"
+      @filter = filter_name
+      @search_term = crawl_type
+
+    # Query search
+    else
+      @crawl_type = "search"
+      @search_term = crawl_type
+    end
+  end
+
+  # Crawls the listings for the query
+  def crawl_listings
+    pages_to_crawl = get_num_pages_per_query.to_i
+    base_url = get_base_query_url
+    
+    # Loop through pages and collect links for each
+    (1..pages_to_crawl).each do |page_num|
+      next_page_html = goto_next_page(base_url, page_num)
+      collect_page_links(next_page_html)
+    end
+  end
+
+  # Collect all the links on the page
+  def collect_page_links(page_html)
+    html = Nokogiri::HTML.parse(page_html)
+    result_rows = html.css("table.search_res").css("tbody").css("tr")
+    
+    # Parse URL and date from each row
+    parsed_result_rows = result_rows.map do |r|
+      link = (@base_url+r.css("a")[0]['href']).split("/keywords")[0]
+      date = Date.parse(r.css("div").select{|e| e.text.include?("Posted - ")}.first.text.gsub("Posted - ", ""))
+      {url: link, posting_date: date}
+    end
+    
+    parse_all_listings(parsed_result_rows)
+    return parsed_result_rows
+  end
+
+  # Parse all of the listings
+  def parse_all_listings(listing_links)
+    found_listings = Array.new
+    listing_links.each do |listing|
+      parser = ClearedJobsNetParser.new(get_page(listing[:url]), listing, @requests)
+      found_listings.push(parser.parse_job)
+    end
+
+    @reporter.report_results(found_listings, listing_links.first)
+  end
+
+  # Gets the number of pages for the query
+  def get_num_pages_per_query
+    # Go to the last page
+    html = Nokogiri::HTML.parse(run_initial_query)
+    if !html.css("div.navbar_bottom").css("a").empty?
+      last_page_link = @base_url+html.css("div.navbar_bottom").css("a").last['href']
+      last_page_html = Nokogiri::HTML.parse(get_page(last_page_link.gsub("//", "/")))
+     
+      # Parse the page numbers in last page
+      return last_page_html.css("div.navbar_bottom").css("strong").text
+    else # Just one page of results
+      return "1"
+    end
+  end
+
+  # Goes to the next page
+  def goto_next_page(base_query_url, num)
+    start_index = (num-1)*25
+
+    # Set the URL for the next page appropriately
+    if start_index == 0 || num == 0
+      next_page_url = base_query_url
+    else
+      next_page_url = base_query_url+"/start/"+start_index.to_s
+    end
+    
+    return get_page(next_page_url)
+  end
+
+  # Open the page
+  def get_page(url, i=0)
+    get_retry(url, @requests, 0)
+  end
+
+  # Get the initial results page
+  def run_initial_query
+    url = get_base_query_url
+    return get_page(url)
+  end
+
+  # Get the base query url depending on type
+  def get_base_query_url
+    if @crawl_type == "all"
+      return @base_url+"search/action/advanced_search/zip_radius/20/keywords/+/city_state_zip/+/security_clearance/+/submit/SEARCH+JOBS/sort/time"
+    elsif @crawl_type == "search"
+      encoded_term = CGI.escape(@search_term)
+      return @base_url+"search/action/advanced_search/zip_radius/20/keywords/"+encoded_term+"/city_state_zip/+/security_clearance/+/submit/SEARCH+JOBS"
+    elsif @crawl_type == "filter"
+      encoded_term = CGI.escape(@search_term)
+      return @base_url+"search/action/advanced_search/keywords/+/"+@filter+"[]/"+encoded_term+"/zip/+/zip_radius/20"
+    elsif @crawl_type == "company_page"
+      return @base_url+"view-employer/employer_id_seo/"+@search_term
+    end
+  end
+
+  # Return JSON
+  def gen_json
+    return @reporter.gen_json
+  end
+end

data/lib/clearedjobsnet/cleared_jobs_net_parser.rb

@@ -0,0 +1,93 @@
+require 'nokogiri'
+require 'pry'
+require 'open-uri'
+
+load 'util/failure_handler.rb'
+
+class ClearedJobsNetParser
+  include FailureHandler
+  def initialize(html, details_hash, requests=nil)
+    @html = Nokogiri::HTML.parse(html)
+    @requests = requests
+    @url = details_hash[:url]
+    @i = 0
+    @posting_date = details_hash[:posting_date]
+  end
+
+  # Parses the job
+  def parse_job
+    begin
+      return {
+        url: @url,
+        html: @html.to_html,
+        posting_date: @posting_date,
+        company_name: company_name,
+        company_listing_link: company_listing_link,
+        required_clearance: required_clearance,
+        location: location,
+        country: country,
+        salary: salary,
+        job_number: job_number,
+        job_title: job_title,
+        job_description: job_description,
+        job_description_plaintext: job_description_plaintext
+      }
+    rescue
+      @i += 1
+      if @i < 10
+        @html = Nokogiri::HTML.parse(get_retry(@url, @requests, @i))
+        parse_job
+      end
+    end
+  end
+
+  # Gets the company name
+  def company_name
+    @html.css("div.view_job_table").css("div.row")[0].css(".left2").text
+  end
+
+  # Get the link to the company page
+  def company_listing_link
+    @html.css("div.view_job_table").css("div.row")[0].css(".left2").css("a")[0]['href']
+  end
+
+  # Gets the clearance level required
+  def required_clearance
+    @html.css("div.view_job_table").css("div.row")[0].css(".clearAll")[1].text
+  end
+
+  # Get the location of work
+  def location
+    @html.css("div.view_job_table").css("div.row")[1].css(".left2").text
+  end
+
+  # Get the country of work
+  def country
+    @html.css("div.view_job_table").css("div.row")[1].css(".right2").text
+  end
+
+  # Get the salary
+  def salary
+    @html.css("div.view_job_table").css("div.row")[2].css(".left2").text.strip.lstrip
+  end
+
+  # Get the job number
+  def job_number
+    @html.css("div.view_job_table").css("div.row")[2].css(".right2").text.strip.lstrip
+  end
+
+  # Get the job title
+  def job_title
+    @html.css("#view_employer").text
+  end
+
+  # Get the job description
+  def job_description
+    @html.css(".view-job-right").to_html
+  end
+
+  # Get the job description without HTML
+  def job_description_plaintext
+    Nokogiri::HTML.parse(job_description.gsub('<br />',"\n").gsub('<br>', "\n").gsub('<br/>', "\n")).text
+  end
+end