scrapin-a-livin 0.1.0

data/test/scripts/diceDump.rb

@@ -0,0 +1,71 @@
+#!/usr/bin/env ruby
+require 'rubygems'
+require 'open-uri'
+require 'hpricot'
+
+# We want to be able to easily pull and update set of listings
+# from the dice website. This script formats the request for us
+# to make things a little easier.
+ENTRIES = 5
+DAYS_BACK = 30
+SEARCH_KEYWORDS = "Software"
+
+# Format the URL to retrieve the current listings from
+# the Dice website
+#
+# http://seeker.dice.com
+# @param where [String, #read] the job location
+# @returns [String] the url request for dice.com
+def get_url(where)
+	"http://seeker.dice.com/jobsearch/servlet/JobSearch" +
+	"?caller=0&LOCATION_OPTION=2&EXTRA_STUFF=1&N=0&Hf=0" +
+	"&Ntk=JobSearchRanking&op=300" +
+	"&values=&FREE_TEXT=#{SEARCH_KEYWORDS}" +
+     "&Ntx=mode+matchall&WHERE=#{where}" +
+	"&WHEREList=#{where}" +
+	"&RADIUS=80.4672" +
+	"&COUNTRY=1525&STAT_PROV=0&METRO_AREA=33.78715899%2C-84.39164034&AREA_CODES=&AC_COUNTRY=1525" +
+	"&TRAVEL=0&TAXTERM=1001" +
+	"&SORTSPEC=0" +
+	"&FRMT=0" +
+	"&DAYSBACK=#{DAYS_BACK}" +
+	"&NUM_PER_PAGE=#{ENTRIES}"
+end
+
+# Write the retrieved data to the file at the specified path.
+#
+# @param filepath [String, #read] the path to the output file
+# @param url [String, #read] the url to use for the request
+def write_query(filepath, url)
+
+	begin
+		# Open the file for writing
+		file = File.new(filepath, "w")
+
+		# Query the page
+		response = open(url, "User-Agent" => "Ruby/#{RUBY_VERSION}",
+					"From" => "email@addr.com",
+					"Referer" => "http://seeker.dice.com/jobsearch/").read
+
+		# Write the data to the file
+		file.write(response)
+
+	ensure
+
+		# Make sure the file is closed
+		file.close
+	end
+
+end
+
+# Query for San Jose
+url = get_url("San+Jose+CA")
+write_query('./querySanJose.html', url)
+
+# Query for San Jose
+url = get_url("Raleigh+NC")
+write_query('./queryRaleigh.html', url)
+
+# Query for San Jose
+url = get_url("Austin+TX")
+write_query('./queryAustin.html', url)

data/test/scripts/hotjobsDump.rb

@@ -0,0 +1,70 @@
+#!/usr/bin/env ruby
+require 'rubygems'
+require 'open-uri'
+require 'hpricot'
+
+# We want to be able to easily pull and update set of listings
+# from the yahoo website. This script formats the request for us
+# to make things a little easier.
+DAYS_BACK = "month"
+SEARCH_KEYWORDS = "Software"
+
+# Format the URL to retrieve the current listings from
+# the Yahoo Hotjobs website
+#
+# http://hotjobs.yahoo.com
+# @param where [String, #read] the job location
+# @returns [String] the url request for dice.com
+def get_url(where)
+
+     "http://hotjobs.yahoo.com/job-search?" +
+	"src=advsearch&pageOp=search&ts=1259353986&" +
+	"kw_search_type=kwany&kw=#{SEARCH_KEYWORDS}&kw_none=&" +
+	"locations=#{where}&country=&locations=&locations=&" +
+	"industry=&industry=&industry=&" +
+	"updated_since=#{DAYS_BACK}&" +
+	"exp_level=&experience_level=&" +
+	"education=&salary[min]=&salary[type]=yearly&" +
+	"commitment=FT&commitment=PT&jobtype=PERM&jobtype=CONT&" +
+	"travel_amount=&company=&" +
+	"source=&email_format=html&email_frequency=1&email_enabled=0&search_jobs=Search+Jobs"
+end
+
+# Write the retrieved data to the file at the specified path.
+#
+# @param filepath [String, #read] the path to the output file
+# @param url [String, #read] the url to use for the request
+def write_query(filepath, url)
+
+	begin
+		# Open the file for writing
+		file = File.new(filepath, "w")
+
+		# Query the page
+		response = open(url, "User-Agent" => "Ruby/#{RUBY_VERSION}",
+					"From" => "email@addr.com",
+					"Referer" => "http://hotjobs.yahoo.com/").read
+
+		# Write the data to the file
+		file.write(response)
+
+	ensure
+
+		# Make sure the file is closed
+		file.close
+	end
+
+end
+
+# Query for San Jose
+url = get_url("San+Jose+CA")
+write_query('./querySanJose.html', url)
+
+
+# Query for San Jose
+url = get_url("Raleigh+NC")
+write_query('./queryRaleigh.html', url)
+
+# Query for San Jose
+url = get_url("Austin+TX")
+write_query('./queryAustin.html', url)

data/test/test_helper.rb

@@ -0,0 +1,8 @@
+require 'rubygems'
+require 'test/unit'
+require 'shoulda'
+# Test Case helper file
+
+$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
+$LOAD_PATH.unshift(File.dirname(__FILE__))
+require 'scrapin-a-livin'

data/test/yahoo_parser_test.rb

@@ -0,0 +1,168 @@
+require 'test_helper'
+require 'ftools'
+
+# Test Case for the yahoo hotjobs listing parser Dice
+class TC_HotjobsSearchParser < Test::Unit::TestCase
+
+	RALEIGH_QUERY = File.dirname(__FILE__) + '/queries/hotjobs/queryRaleigh.html'
+	AUSTIN_QUERY = File.dirname(__FILE__) + '/queries/hotjobs/queryAustin.html'
+	SANJOSE_QUERY = File.dirname(__FILE__) + '/queries/hotjobs/querySanJose.html'
+
+	# Test for the listing header
+	def test_raleigh
+
+		# Read the query data from the file
+		data = check_query(RALEIGH_QUERY)
+
+		# Parse the listings from the query
+		listings = HotjobsSearch.parse_listings(data)
+
+		# Go through each listing
+		assert_equal("Embedded Software Engineer", listings[0].title)
+		assert_equal("http://hotjobs.yahoo.com/job-JNEJ91O9CZA?source=SRP", listings[0].link)
+		assert_equal("Sirit Technology", listings[0].company)
+		assert_equal("http://hotjobs.yahoo.com/careers-786030-Sirit_Technology", listings[0].company_link)
+		assert_equal("Morrisville, NC", listings[0].location)
+		assert_equal("Oct 28", listings[0].date)
+
+		assert_equal("MES Java Software Engineer", listings[1].title)
+		assert_equal("http://hotjobs.yahoo.com/job-JDZWR4DRM00?source=SRP", listings[1].link)
+		assert_equal("Werum Software & Systems America", listings[1].company)
+		assert_equal("http://hotjobs.yahoo.com/careers-616438-Werum_Software___Systems_America", listings[1].company_link)
+		assert_equal("Cary, NC", listings[1].location)
+		assert_equal("Sep 29", listings[1].date)
+
+		assert_equal("Software Developer", listings[2].title)
+		assert_equal("http://hotjobs.yahoo.com/job-JQGSVB1Y99F?source=SRP", listings[2].link)
+		assert_equal("CTG", listings[2].company)
+		assert_equal("http://hotjobs.yahoo.com/careers-601569-CTG", listings[2].company_link)
+		assert_equal("Durham, NC", listings[2].location)
+		assert_equal("Oct 30", listings[2].date)
+
+		assert_equal("Software Engineer - Windows Installer Developer, MSI, WiX, WiM", listings[3].title)
+		assert_equal("http://hotjobs.yahoo.com/job-J9WH2JP6WO7?source=SRP", listings[3].link)
+		assert_equal("CyberCoders", listings[3].company)
+		assert_equal("http://hotjobs.yahoo.com/careers-577525-CyberCoders", listings[3].company_link)
+		assert_equal("Raleigh, NC", listings[3].location)
+		assert_equal("Nov 27", listings[3].date)
+
+		assert_equal("C# Developer, Software Developer, SAAS, ASP.Net, SQL Server, SQL, .Net, WCF, JavaScript", listings[4].title)
+		assert_equal("http://hotjobs.yahoo.com/job-J5PKCPM1QM2?source=SRP", listings[4].link)
+		assert_equal("CyberCoders", listings[4].company)
+		assert_equal("http://hotjobs.yahoo.com/careers-577525-CyberCoders", listings[4].company_link)
+		assert_equal("Raleigh, NC", listings[4].location)
+		assert_equal("Nov 27", listings[4].date)
+	end
+
+	# Test for parsing a listing
+	def test_austin
+
+		# Read the query data
+		data = check_query(AUSTIN_QUERY)
+
+		# Parse the listings from the query
+		listings = HotjobsSearch.parse_listings(data)
+
+		# Go through each listing
+		assert_equal("Software Program Manager", listings[0].title)
+		assert_equal("http://hotjobs.yahoo.com/job-JSFNSIFFNCA?source=SRP", listings[0].link)
+		assert_equal("Zebra Imaging", listings[0].company)
+		assert_equal("http://hotjobs.yahoo.com/careers-627739-Zebra_Imaging", listings[0].company_link)
+		assert_equal("Austin, TX", listings[0].location)
+		assert_equal("Nov 19", listings[0].date)
+
+		assert_equal("Client Project Manager - Software", listings[1].title)
+		assert_equal("http://hotjobs.yahoo.com/job-JM1F32JSA7R?source=SRP", listings[1].link)
+		assert_equal("Digital Cheetah Solutions, Inc.", listings[1].company)
+		assert_equal("http://hotjobs.yahoo.com/careers-577804-Digital_Cheetah_Solutions__Inc_", listings[1].company_link)
+		assert_equal("Austin, TX", listings[1].location)
+		assert_equal("Nov 08", listings[1].date)
+
+		assert_equal("Software Quality Assurance Lead", listings[2].title)
+		assert_equal("http://hotjobs.yahoo.com/job-JL5SOECPQKF?source=SRP", listings[2].link)
+		assert_equal("Zebra Imaging", listings[2].company)
+		assert_equal("http://hotjobs.yahoo.com/careers-627739-Zebra_Imaging", listings[2].company_link)
+		assert_equal("Austin, TX", listings[2].location)
+		assert_equal("Oct 06", listings[2].date)
+
+		assert_equal("Operating System Software Test Specialist", listings[3].title)
+		assert_equal("http://hotjobs.yahoo.com/job-J3KBF8EADAO?source=SRP", listings[3].link)
+		assert_equal("CTG", listings[3].company)
+		assert_equal("http://hotjobs.yahoo.com/careers-601569-CTG", listings[3].company_link)
+		assert_equal("Austin, TX", listings[3].location)
+		assert_equal("Nov 12", listings[3].date)
+
+		assert_equal("Software Engineer", listings[4].title)
+		assert_equal("http://hotjobs.yahoo.com/job-JTXD1K5GAZ2?source=SRP", listings[4].link)
+		assert_equal("Troux Technologies", listings[4].company)
+		assert_equal("http://hotjobs.yahoo.com/careers-788124-Troux_Technologies", listings[4].company_link)
+		assert_equal("Austin, TX", listings[4].location)
+		assert_equal("Nov 05", listings[4].date)
+
+	end
+
+	# Test for parsing a listing
+	def test_sanjose
+
+		# Read the query data
+		data = check_query(SANJOSE_QUERY)
+
+		# Parse the listings from the query
+		listings = HotjobsSearch.parse_listings(data)
+
+		# Go through each listing
+		assert_equal("Software Engineer/Carrier CE", listings[0].title)
+		assert_equal("http://hotjobs.yahoo.com/job-J2VW8Y7GMAH?source=SRP", listings[0].link)
+		assert_equal("Atheros Communications, Inc.", listings[0].company)
+		assert_equal("http://hotjobs.yahoo.com/careers-552561-Atheros_Communications__Inc_", listings[0].company_link)
+		assert_equal("Santa Clara, CA", listings[0].location)
+		assert_equal("Nov 12", listings[0].date)
+
+		assert_equal("EMBEDDED SOFTWARE ENGINEER", listings[1].title)
+		assert_equal("http://hotjobs.yahoo.com/job-JD285WTGJLP?source=SRP", listings[1].link)
+		assert_equal("NVIDIA Corporation", listings[1].company)
+		assert_equal("http://hotjobs.yahoo.com/careers-601859-NVIDIA_Corporation", listings[1].company_link)
+		assert_equal("Santa Clara, CA", listings[1].location)
+		assert_equal("Oct 23", listings[1].date)
+
+		assert_equal("EMBEDDED SOFTWARE ENGINEER", listings[2].title)
+		assert_equal("http://hotjobs.yahoo.com/job-J9LJP1J38EB?source=SRP", listings[2].link)
+		assert_equal("NVIDIA Corporation", listings[2].company)
+		assert_equal("http://hotjobs.yahoo.com/careers-601859-NVIDIA_Corporation", listings[2].company_link)
+		assert_equal("Santa Clara, CA", listings[2].location)
+		assert_equal("Oct 23", listings[2].date)
+
+		assert_equal("Platform Bringup Software Engineer", listings[3].title)
+		assert_equal("http://hotjobs.yahoo.com/job-J6YYFVVFK7P?source=SRP", listings[3].link)
+		assert_equal("Aruba Networks", listings[3].company)
+		assert_equal("http://hotjobs.yahoo.com/careers-562030-Aruba_Networks", listings[3].company_link)
+		assert_equal("Sunnyvale, CA", listings[3].location)
+		assert_equal("Nov 14", listings[3].date)
+
+		assert_equal("Staff Software Development Engineer", listings[4].title)
+		assert_equal("http://hotjobs.yahoo.com/job-JDG4ZD5RP59?source=SRP", listings[4].link)
+		assert_equal("IDT", listings[4].company)
+		assert_equal("http://hotjobs.yahoo.com/careers-577851-IDT", listings[4].company_link)
+		assert_equal("San Jose, CA", listings[4].location)
+		assert_equal("Nov 19", listings[4].date)
+	end
+
+	private
+
+	# Make sure the query file exists
+	#
+	# @param path [String, #read] the path to the input file
+	# @returns [String] the data from the input file
+	def check_query(path)
+
+		# Check to make sure the query exists and is readable
+		if !File.exists?(path) || !File.readable?(path)
+
+			raise ArgumentError.new "Invalid Query: \"#{path}\""
+		end
+
+		# Open the file and read the contents
+		File.open(path, "r").read
+	end
+
+end

metadata

@@ -0,0 +1,110 @@
+--- !ruby/object:Gem::Specification 
+name: scrapin-a-livin
+version: !ruby/object:Gem::Version 
+  version: 0.1.0
+platform: ruby
+authors: 
+- Kevin S Kirkup
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2009-11-28 00:00:00 -05:00
+default_executable: 
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: hpricot
+  type: :runtime
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: "0.6"
+    version: 
+- !ruby/object:Gem::Dependency 
+  name: thoughtbot-shoulda
+  type: :development
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: "0"
+    version: 
+description: |
+  Are you in the technology industry?
+  The you have probably lost a job during your career.
+  This script helps you scrape the most common job sites for your search criteria,
+  and save the lists on you local computer.
+  
+  You can then use other tools to display updates, send out resumes or search your LinkedIn account
+  to see who you may know at that company.
+  
+  Please feel free to contribute make. All suggestions are welcome.
+  Hopefully this will help you find the career you are looking for.
+
+email: kevin.kirkup@gmail.com
+executables: []
+
+extensions: []
+
+extra_rdoc_files: 
+- LICENSE
+- README.markdown
+files: 
+- .document
+- .gitignore
+- LICENSE
+- README.markdown
+- Rakefile
+- VERSION
+- lib/dice/dice_search.rb
+- lib/generic/listing.rb
+- lib/scrapin-a-livin.rb
+- lib/yahoo/hotjobs.rb
+- test/dice_parser_test.rb
+- test/queries/dice/queryAustin.html
+- test/queries/dice/queryRaleigh.html
+- test/queries/dice/querySanJose.html
+- test/queries/hotjobs/queryAustin.html
+- test/queries/hotjobs/queryRaleigh.html
+- test/queries/hotjobs/querySanJose.html
+- test/scripts/diceDump.rb
+- test/scripts/hotjobsDump.rb
+- test/test_helper.rb
+- test/yahoo_parser_test.rb
+has_rdoc: true
+homepage: http://github.com/angrytuna/scrapin-a-livin
+licenses: []
+
+post_install_message: 
+rdoc_options: 
+- --charset=UTF-8
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 1.3.5
+signing_key: 
+specification_version: 3
+summary: Scrape to find a job
+test_files: 
+- test/scripts/diceDump.rb
+- test/scripts/hotjobsDump.rb
+- test/yahoo_parser_test.rb
+- test/dice_parser_test.rb
+- test/test_helper.rb