tsjobcrawler 0.1

data/lib/tsjobcrawler.rb

@@ -0,0 +1,52 @@
+require 'json'
+load 'securityclearedjobscom/security_cleared_jobs_com_crawler.rb'
+load 'clearancejobscom/clearance_jobs_com_crawler.rb'
+load 'clearedjobsnet/cleared_jobs_net_crawler.rb'
+load 'clearedjobsnet/get_all_cleared_jobs.rb'
+
+# Crawls all the jobs that require clearance
+class TSJobCrawler
+  def initialize(search_term, requests=nil, cm_hash=nil)
+    @search_term = search_term
+    @requests = requests
+    @cm_hash = cm_hash
+    @output = Array.new
+  end
+
+  # Crawl all of the listing sites
+  def crawl_jobs
+    security_cleared_jobs_com
+    clearance_jobs_com
+    cleared_jobs_net
+  end
+
+  def security_cleared_jobs_com
+    c = SecurityClearedJobsComCrawler.new(@search_term, @requests, @cm_hash)
+    c.crawl
+    @output += JSON.parse(c.gen_json) if @cm_hash == nil
+  end
+
+  def clearance_jobs_com
+    c = ClearanceJobsComCrawler.new(@search_term, @requests, @cm_hash)
+    c.crawl
+    @output += JSON.parse(c.gen_json) if @cm_hash == nil
+  end
+
+  def cleared_jobs_net
+    if @search_term == nil
+      g = GetAllClearedJobs.new(@requests, @cm_hash)
+      g.crawl
+      @output += JSON.parse(g.gen_json) if @cm_hash == nil
+    else # Scrape by search term
+      c = ClearedJobsNetCrawler.new(@search_term, nil, @requests, @cm_hash)
+      c.crawl_listings
+      @output += JSON.parse(c.gen_json) if @cm_hash == nil
+    end
+  end
+
+  # Generate output
+  def gen_json
+    JSON.pretty_generate(@output)
+  end
+end
+

data/lib/util/failure_handler.rb

@@ -0,0 +1,22 @@
+require 'requestmanager'
+require 'nokogiri'
+require 'open-uri'
+
+module FailureHandler
+  def get_retry(url, requests, i=0)
+    puts "crawling "+url
+    begin
+      if requests
+        return requests.get_page(url)
+      else
+        return File.read(open(url.gsub("[", "%5B").gsub("]", "%5D")))
+      end
+    rescue
+      if i < 10
+        i+=1
+        sleep(i*rand(1..10))
+        get_retry(url, requests, i)
+      end
+    end
+  end
+end

data/tsjobcrawler.gemspec

@@ -0,0 +1,27 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+
+Gem::Specification.new do |spec|
+  spec.name          = "tsjobcrawler"
+  spec.version       = '0.1'
+  spec.authors       = ["M. C. McGrath"]
+  spec.email         = ["shidash@shidash.com"]
+
+  spec.summary       = %q{Crawls job listing websites for jobs requiring security clearance.}
+  spec.description   = %q{Crawls job listing websites for jobs requiring security clearance.}
+  spec.homepage      = "https://github.com/TransparencyToolkit/TSJobCrawler"
+
+  spec.files         = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
+  spec.bindir        = "exe"
+  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.require_paths = ["lib"]
+
+  spec.add_development_dependency "bundler", "~> 1.10"
+  spec.add_development_dependency "rake", "~> 10.0"
+  spec.add_runtime_dependency "nokogiri"
+  spec.add_runtime_dependency "requestmanager"
+  spec.add_runtime_dependency "harvesterreporter"
+  spec.add_runtime_dependency "pry"
+  spec.add_runtime_dependency "headless"
+end

metadata

@@ -0,0 +1,162 @@
+--- !ruby/object:Gem::Specification
+name: tsjobcrawler
+version: !ruby/object:Gem::Version
+  version: '0.1'
+platform: ruby
+authors:
+- M. C. McGrath
+autorequire: 
+bindir: exe
+cert_chain: []
+date: 2017-03-13 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.10'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.10'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+- !ruby/object:Gem::Dependency
+  name: nokogiri
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: requestmanager
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: harvesterreporter
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: pry
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: headless
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Crawls job listing websites for jobs requiring security clearance.
+email:
+- shidash@shidash.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".rspec"
+- COPYING
+- Gemfile
+- README.md
+- bin/console
+- bin/setup
+- lib/clearancejobscom/clearance_jobs_com_crawler.rb
+- lib/clearancejobscom/clearance_jobs_com_parser.rb
+- lib/clearedjobsnet/cleared_jobs_net_crawler.rb
+- lib/clearedjobsnet/cleared_jobs_net_parser.rb
+- lib/clearedjobsnet/get_all_cleared_jobs.rb
+- lib/clearedjobsnet/terms/clearance_levels.json
+- lib/clearedjobsnet/terms/company_names.json
+- lib/clearedjobsnet/terms/country_names.json
+- lib/clearedjobsnet/terms/search_terms.json
+- lib/securityclearedjobscom/security_cleared_jobs_com_crawler.rb
+- lib/securityclearedjobscom/security_cleared_jobs_com_parser.rb
+- lib/tsjobcrawler.rb
+- lib/util/failure_handler.rb
+- tsjobcrawler.gemspec
+homepage: https://github.com/TransparencyToolkit/TSJobCrawler
+licenses: []
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.4.8
+signing_key: 
+specification_version: 4
+summary: Crawls job listing websites for jobs requiring security clearance.
+test_files: []
+has_rdoc: