tsjobcrawler 0.1

data/lib/clearedjobsnet/get_all_cleared_jobs.rb

@@ -0,0 +1,77 @@
+require 'json'
+require 'headless'
+require 'requestmanager'
+load 'clearedjobsnet/cleared_jobs_net_crawler.rb'
+
+# Get as many jobs as possible
+class GetAllClearedJobs
+  def initialize(requests, cm_hash)
+    @output = Array.new
+    @requests = requests
+    @cm_hash
+  end
+
+  # Crawl through many options
+  def crawl
+   get_first_1000
+   get_by_clearance
+   get_by_country
+   get_by_company
+   get_by_searchterm
+  end
+
+  # Get the most recent jobs from blank search
+  def get_first_1000
+    start_crawler("all")
+  end
+
+  # Crawl by security clearance
+  def get_by_clearance
+    clearance_levels = JSON.parse(File.read("clearedjobsnet/terms/clearance_levels.json"))
+    crawl_each(clearance_levels, "security_clearance")
+  end
+
+  # Crawl each country
+  def get_by_country
+    country_names = JSON.parse(File.read("clearedjobsnet/terms/country_names.json"))
+    crawl_each(country_names, "country")
+  end
+
+  # Crawl company pages
+  def get_by_company
+    company_names = JSON.parse(File.read("clearedjobsnet/terms/company_names.json"))
+    crawl_each(company_names, "company_page")
+  end
+
+  # Crawl search term list
+  def get_by_searchterm
+    search_terms = JSON.parse(File.read("clearedjobsnet/terms/search_terms.json"))
+    crawl_each(search_terms)
+  end
+
+  
+  # Crawl each item
+  def crawl_each(term_list, filter_name=nil)
+    term_list.each do |term|
+      start_crawler(term, filter_name)
+    end
+  end
+
+  # Start the crawler
+  def start_crawler(search_term, filter=nil)
+    c = ClearedJobsNetCrawler.new(search_term, filter, @requests, @cm_hash)
+    c.crawl_listings
+    save_listings(c.gen_json)
+  end
+
+  # Save unique listings in output
+  def save_listings(listings)
+    @output = @output | JSON.parse(listings)
+  end
+
+  # Generates output JSON
+  def gen_json
+    return JSON.pretty_generate(@output)
+  end
+end
+

data/lib/clearedjobsnet/terms/clearance_levels.json

@@ -0,0 +1,15 @@
+[
+    "DoJ",
+    "IRS",
+    "Secret",
+    "Top Secret",
+    "Top Secret / SCI",
+    "Top Secret / SCI + Poly",
+    "Top Secret / SCI + CI Poly",
+    "Top Secret / SCI + Full Scope Poly",
+    "DHS",
+    "DoE",
+    "Public Trust",
+    "Confidential",
+    "Security Clearance Required"
+]

data/lib/clearedjobsnet/terms/company_names.json

@@ -0,0 +1,17 @@
+[
+    "sos-international-llc-0701",
+    "leidos-0062",
+    "caci-0108",
+    "northrop-grumman-0388",
+    "booz-allen-hamilton-0816",
+    "raytheon--0833",
+    "mantech-international-155811",
+    "csra-0194",
+    "engility-0380",
+    "hewlett-packard-enterprise-company-0212",
+    "pae-1023",
+    "jacobs-0868",
+    "los-alamos-national-laboratory-200019",
+    "amazon-web-services-0834",
+    "deloitte-0086"
+]

data/lib/clearedjobsnet/terms/country_names.json

@@ -0,0 +1,202 @@
+[
+  "Canada",
+  "United States",
+  "United Kingdom",
+  "Australia",
+  "Brazil",
+  "Afghanistan",
+  "Albania",
+  "American Samoa",
+  "Andorra",
+  "Angola",
+  "Anguilla",
+  "Antigua and Barbuda",
+  "Argentina",
+  "Armenia",
+  "Aruba",
+  "Austria",
+  "Azerbaijan Republic",
+  "Bahamas",
+  "Bahrain",
+  "Bangladesh",
+  "Barbados",
+  "Belarus",
+  "Belgium",
+  "Belize",
+  "Benin",
+  "Bermuda",
+  "Bhutan",
+  "Bolivia",
+  "Bosnia and Herzegovina",
+  "Botswana",
+  "British Virgin Islands",
+  "Brunei Darussalam",
+  "Bulgaria",
+  "Burkina Faso",
+  "Burma",
+  "Burundi",
+  "Cambodia",
+  "Cameroon",
+  "Cape Verde Islands",
+  "Cayman Islands",
+  "Central African Republic",
+  "Chile",
+  "Colombia",
+  "Comoros",
+  "Cook Islands",
+  "Costa Rica",
+  "Cote d Ivoire (Ivory Coast)",
+  "Croatia, Republic of",
+  "Cuba",
+  "Cyprus",
+  "Czech Republic",
+  "Denmark",
+  "Djibouti",
+  "Dominica",
+  "Dominican Republic",
+  "Ecuador",
+  "El Salvador",
+  "Equatorial Guinea",
+  "Eritrea",
+  "Estonia",
+  "Falkland Islands (Islas Malvinas)",
+  "Fiji",
+  "Finland",
+  "France",
+  "French Guiana",
+  "French Polynesia",
+  "Gabon Republic",
+  "Gambia",
+  "Georgia",
+  "Germany",
+  "Ghana",
+  "Gibraltar",
+  "Greece",
+  "Greenland",
+  "Grenada",
+  "Guadeloupe",
+  "Guam",
+  "Guatemala",
+  "Guernsey",
+  "Guinea",
+  "Guinea-Bissau",
+  "Guyana",
+  "Haiti",
+  "Honduras",
+  "Hong Kong",
+  "Hungary",
+  "Iceland",
+  "Indonesia",
+  "Iraq",
+  "Ireland",
+  "Israel",
+  "Italy",
+  "Jamaica",
+  "Jan Mayen",
+  "Japan",
+  "Jersey",
+  "Jordan",
+  "Kazakhstan",
+  "Kiribati",
+  "Korea, South",
+  "Kosovo",
+  "Kuwait",
+  "Kyrgyzstan",
+  "Laos",
+  "Latvia",
+  "Liechtenstein",
+  "Lithuania",
+  "Luxembourg",
+  "Macau",
+  "Macedonia",
+  "Madagascar",
+  "Malawi",
+  "Malaysia",
+  "Maldives",
+  "Malta",
+  "Marshall Islands",
+  "Martinique",
+  "Mauritania",
+  "Mauritius",
+  "Mayotte",
+  "Mexico",
+  "Moldova",
+  "Monaco",
+  "Mongolia",
+  "Montserrat",
+  "Mozambique",
+  "Namibia",
+  "Nauru",
+  "Nepal",
+  "Netherlands",
+  "Netherlands Antilles",
+  "New Caledonia",
+  "New Zealand",
+  "Nicaragua",
+  "Niger",
+  "Niue",
+  "Norway",
+  "Palau",
+  "Panama",
+  "Papua New Guinea",
+  "Paraguay",
+  "Peru",
+  "Philippines",
+  "Poland",
+  "Portugal",
+  "Puerto Rico",
+  "Qatar",
+  "Romania",
+  "Rwanda",
+  "Saint Helena",
+  "Saint Kitts-Nevis",
+  "Saint Lucia",
+  "Saint Pierre and Miquelon",
+  "Saint Vincent and the Grenadines",
+  "San Marino",
+  "Saudi Arabia",
+  "Senegal",
+  "Seychelles",
+  "Sierra Leone",
+  "Singapore",
+  "Slovakia",
+  "Slovenia",
+  "Solomon Islands",
+  "South Africa",
+  "Spain",
+  "Sri Lanka",
+  "Sudan",
+  "Suriname",
+  "Svalbard",
+  "Swaziland",
+  "Sweden",
+  "Switzerland",
+  "Tahiti",
+  "Taiwan",
+  "Tajikistan",
+  "Tanzania",
+  "Thailand",
+  "Togo",
+  "Tonga",
+  "Trinidad and Tobago",
+  "Tunisia",
+  "Turkey",
+  "Turkmenistan",
+  "Turks and Caicos Islands",
+  "Tuvalu",
+  "Uganda",
+  "Ukraine",
+  "United Arab Emirates",
+  "Uruguay",
+  "Uzbekistan",
+  "Vanuatu",
+  "Vatican City State",
+  "Venezuela",
+  "Virgin Islands (U.S.)",
+  "Wallis and Futuna",
+  "Western Sahara",
+  "Western Samoa",
+  "Zimbabwe",
+  "Yugoslavia",
+  "Zambia"
+]

data/lib/clearedjobsnet/terms/search_terms.json

@@ -0,0 +1,27 @@
+[
+    "COMINT",
+    "OSINT",
+    "HUMINT",
+    "SIGINT",
+    "MASINT",
+    "GEOINT",
+    "IMINT",
+    "intelligence",
+    "analyst",
+    "target",
+    "linguist",
+    "research",
+    "software",
+    "network",
+    "supply",
+    "engineer",
+    "finance",
+    "cyber",
+    "geographic",
+    "manager",
+    "project",
+    "program",
+    "hardware",
+    "technician",
+    "administrative"
+]

data/lib/securityclearedjobscom/security_cleared_jobs_com_crawler.rb

@@ -0,0 +1,93 @@
+require 'json'
+require 'nokogiri'
+require 'open-uri'
+require 'cgi'
+require 'pry'
+require 'requestmanager'
+require 'headless'
+require 'harvesterreporter'
+
+load 'securityclearedjobscom/security_cleared_jobs_com_parser.rb'
+load 'util/failure_handler.rb'
+
+class SecurityClearedJobsComCrawler
+  include FailureHandler
+  def initialize(search_term, requests=nil, cm_hash=nil)
+    @search_term = search_term
+    @requests = requests
+    @site_url = "https://www.securityclearedjobs.com"
+    @query_base_url = set_base_url
+  
+    @reporter = HarvesterReporter.new(cm_hash)
+  end
+
+  # Set the base url for the query
+  def set_base_url
+    if @search_term == nil
+      return @site_url+"/searchjobs/?countrycode=GB"
+    else
+      return @site_url+"/searchjobs/?countrycode=GB&Keywords="+CGI.escape(@search_term)
+    end
+  end
+
+  # Get the page
+  def get_page(url)
+    get_retry(url, @requests, 0)
+  end
+
+  # Get the total pagecount
+  def get_total_pagecount
+    initial_page = Nokogiri::HTML.parse(load_next_page(1))
+    navbar = initial_page.css(".paginator__item").last
+    last_page_link = navbar.css("a")[0]['href'] if navbar
+
+    # Handle case of there just being one page
+    if last_page_link
+      page_count = last_page_link.split("&Page=")[1].to_i
+      page_count == 0 ? (return 1) : (return page_count)
+    end
+  end
+
+  # Load the next page
+  def load_next_page(page_num)
+    next_page_url = @query_base_url + "&Page="+page_num.to_s
+    return get_page(next_page_url)
+  end
+
+  # Save the result links on a page
+  def save_result_links(page)
+    html = Nokogiri::HTML.parse(page)
+    return html.css(".lister__header").css("a").map{|e| @site_url+e['href']}
+  end
+
+  # Parse all the listings on a single page
+  def parse_listings(page)
+    listing_links = save_result_links(page)
+    found_listings = Array.new
+    
+    listing_links.each do |listing|
+      parser = SecurityClearedJobsComParser.new(listing, get_page(listing), @requests)
+      parsed_listing = parser.parse
+      found_listings.push(parsed_listing) if parsed_listing
+    end
+    
+    @reporter.report_results(found_listings, listing_links.first)
+  end
+
+  # Crawls all of the listings
+  def crawl
+    total_pagecount = get_total_pagecount
+    
+    # Load each page
+    (1..total_pagecount.to_i).each do |page_num|
+      next_page = load_next_page(page_num)
+      parse_listings(next_page)
+    end
+  end
+
+  # Output JSON
+  def gen_json
+    return @reporter.gen_json
+  end
+end
+

data/lib/securityclearedjobscom/security_cleared_jobs_com_parser.rb

@@ -0,0 +1,115 @@
+require 'nokogiri'
+require 'open-uri'
+require 'pry'
+
+load 'util/failure_handler.rb'
+
+class SecurityClearedJobsComParser
+  include FailureHandler
+  def initialize(url, html, requests=nil)
+    @url = url
+    @requests = requests
+    @html = html
+    @page = Nokogiri::HTML.parse(@html)
+  end
+
+  # Parse the job listing
+  def parse
+    begin
+      return {
+        url: @url,
+        company_name: company_name,
+        location: location,
+        salary: salary,
+        posting_date: posting_date,
+        closing_date: closing_date,
+        job_number: job_number,
+        contact_person: contact_person,
+        employment_status: employment_status,
+        required_clearance: required_clearance,
+        job_category: job_category,
+        job_title: job_title,
+        job_description: job_description,
+        job_description_plaintext: job_description_plaintext,
+        html: @html
+      }
+    rescue
+      @i += 1
+      if @i < 10
+        @html = Nokogiri::HTML.parse(get_retry(@url, @requests, @i))
+        parse
+      end
+    end
+  end
+
+  # Get the name of the company
+  def company_name
+    @page.css("div.cf[itemprop='hiringOrganization']")[0].css("span[itemprop='name']").text
+  end
+
+  # Get the location
+  def location
+    get_element("Location")
+  end
+
+  # Get the salary
+  def salary
+    get_element("Salary")
+  end
+
+  # Get the posting date
+  def posting_date
+    Date.parse(get_element("Posted"))
+  end
+
+  # Get the date it closes
+  def closing_date
+    Date.parse(get_element("Closes"))
+  end
+
+  # Get the job number
+  def job_number
+    get_element("Ref")
+  end
+
+  # Get the contact person
+  def contact_person
+    get_element("Contact")
+  end
+
+  # Get the employment status
+  def employment_status
+    get_element("Job Type")
+  end
+
+  # Gets the clearance level required
+  def required_clearance
+    get_element("Clearance Level").split(", ")
+  end
+
+  # Gets the sector of the job
+  def job_category
+    get_element("Sector").split(", ")
+  end
+
+  # Get the job title
+  def job_title
+    @page.css("h1[itemprop='title']").text
+  end
+
+  # Get the job description
+  def job_description
+    @page.css("div[itemprop='description']").to_html
+  end
+
+  # Get the job description without html
+  def job_description_plaintext
+    Nokogiri::HTML.parse(job_description.gsub('<br />',"\n").gsub('<br>', "\n").gsub('<br/>', "\n")).text
+  end
+
+  # Get the element for the field
+  def get_element(field_name)
+    element = @page.css("div.cf").select{|d| d.text.include?(field_name)}
+    element[0].css("dd").text.strip.lstrip.gsub(/\s+/, " ") if !element.empty?
+  end
+end