scrapin-a-livin 0.1.0 → 0.1.1

data/README.markdown

@@ -1,10 +1,7 @@
 # scrapin-a-livin
 
 This script helps you screen scrape the most common job sites and save the lists on you local computer.
-
-You can then use other tools to display updates, send out resumes or search your LinkedIn account
-to see who you may know at that company.
-
+You can then use other tools to display updates, send out resumes or search your LinkedIn account to see who you may know at that company.
 Please feel free to contribute make. All suggestions are welcome.
 Hopefully this will help you find the career you are looking for.
 
@@ -13,12 +10,34 @@ http://www.igvita.com/2007/02/04/ruby-screen-scraper-in-60-seconds/
 
 ## Quick links
 
-  *[Wiki](http://wiki.github.com/angrytuna/scrapin-a-livin)
-  *[Bugs](http://github.com/angrytuna/scrapin-a-livin/issues)
+  * [Wiki](http://wiki.github.com/angrytuna/scrapin-a-livin)
+  * [Bugs](http://github.com/angrytuna/scrapin-a-livin/issues)
+
+## Installing
+
+    # Install the gem
+    $ sudo gem install scrapin-a-livin
+
+## Using
+
+To use the require the scrapin-a-livin library and then use the static methods for the site you want data from.
+
+    require 'scrapin-a-livin'
+
+    # Create a url for the request
+    url = HotjobsSearch.query("Raleigh+NC", "Software")
+
+    # Use the query to retrieve the job listings
+    listings = HotjobsSearch.get_listings(url)
 
-## How To
+    # Now you can use the job listing information
+    listings.each { |job|
+      puts job.title
+      puts job.company
+    }
 
-A
+The current support for creating the html query for the both Dice and Yahoo Hotjobs is quite limited.
+If you would like to have a more advanced search, you can go the either site, create the search using the advanced search editor, and them copy the url from the resulting query.  Then just substitute this string in the code above to retrieve the listings.
 
 ## Note on Patches/Pull Requests
  

data/VERSION

@@ -1 +1 @@
-0.1.0
+0.1.1

data/lib/dice/dice.rb

@@ -0,0 +1,122 @@
+#!/usr/bin/env ruby
+require 'rubygems'
+require 'open-uri'
+require 'hpricot'
+
+# Class to screen scrape the dice website
+#
+# http://seeker.dice.com
+class DiceSearch
+
+	# Constants
+	DEBUG = false
+	TITLE_CELL = 2
+	COMPANY_CELL = 3
+	LOCATION_CELL = 4
+	DATE_CELL = 5
+	CELL_COUNT = 6
+	DICE_LINK = "http://seeker.dice.com"
+
+	# Parse the provided query data
+	#
+	# @param query [String, #read] the html web page data
+	# @returns [JobListings] an array of job listings
+	def self.parse_listings(query)
+
+		# Create the listings
+		listings = Array.new
+
+		# Filter the data with Hpricot
+		doc = Hpricot(query)
+
+		# Get the table
+		table = (doc/"//table[@class=summary]")
+		
+		# Get the rows
+		rows = (table/"tr")
+		
+		# Retrieve the table rows that contain the job listings
+		rows.each { |row|
+		
+			# Get the individual cells
+			cells = (row/"td")
+		
+			# If this is a job listing
+			if cells.size == CELL_COUNT
+
+				# Get the fields
+				name = (cells[TITLE_CELL]/"a").inner_html
+				link = DICE_LINK + (cells[TITLE_CELL]/"a").attr("href")
+				company = (cells[COMPANY_CELL]/"a").inner_html
+				company_link = DICE_LINK + (cells[COMPANY_CELL]/"a").attr("href")
+				location = cells[LOCATION_CELL].inner_html
+				date = cells[DATE_CELL].inner_html
+
+				if DEBUG
+					puts "Row: count #{cells.size}"
+					puts "Name: #{name}"
+					puts "Link: #{link}"
+					puts "Company: #{company}"
+					puts "Company Link: #{company_link}"
+					puts "Location: #{location}"
+					puts "Date: #{date}"
+				end
+
+				# Create the job listing
+				listings << JobListing.new(name, link, company, company_link, location, date)
+
+			end
+
+		}
+
+		# Return the listings
+		return listings
+
+	end
+
+	# Retrieve the job listings
+	#
+	# @param url [String, #read] the url used to query the data
+	# @param [JobListings] an array of job listings
+	def self.get_listings(url)
+
+		# Read the data from the url
+		response = open(url, "User-Agent" => "Ruby/#{RUBY_VERSION}",
+					"From" => "email@addr.com",
+					"Referer" => "http://hotjobs.yahoo.com/").read
+
+		# Parse the listings from the query
+		parse_listings(response)
+
+	end
+
+	# Query dice for html code for the query
+	#
+	# @param location [String, #read] the location to search
+	# @param keywords [String, #read] keywords to use for the search
+	# @param days_back [String, #read] how long ago to search
+	# @param num_entries [String, #read] the number of entries to request
+	def self.query(location, keywords, days_back, num_entries)
+
+		# The search URL
+		url = "http://seeker.dice.com/jobsearch/servlet/JobSearch" +
+			 "?caller=0&LOCATION_OPTION=2&EXTRA_STUFF=1&N=0&Hf=0" +
+			 "&Ntk=JobSearchRanking&op=300" +
+			 "&values=&FREE_TEXT=#{keywords}" +
+			 "&Ntx=mode+matchall&WHERE=#{location}" +
+			 "&WHEREList=#{location}" +
+			 "&RADIUS=80.4672" +
+			 "&COUNTRY=1525&STAT_PROV=0&METRO_AREA=33.78715899%2C-84.39164034&AREA_CODES=&AC_COUNTRY=1525" +
+			 "&TRAVEL=0&TAXTERM=1001" +
+			 "&SORTSPEC=0" +
+			 "&FRMT=0" +
+			 "&DAYSBACK=#{days_back}" +
+			 "&NUM_PER_PAGE=#{num_entries}"
+
+		# Read the data from the url
+		open(@url, "User-Agent" => "Ruby/#{RUBY_VERSION}",
+			"From" => "email@addr.com",
+			"Referer" => "http://seeker.dice.com/jobsearch/").read
+	end
+
+end

data/lib/scrapin-a-livin.rb

@@ -1,43 +1,6 @@
 #!/usr/bin/env ruby
 require 'generic/listing.rb'
-require 'dice/dice_search.rb'
+require 'dice/dice.rb'
 require 'yahoo/hotjobs.rb'
 
 # Helper file to include the available libraries
-
-# Main entry
-if $0 == (__FILE__)
-
-#  # Check the arguments
-#  case ARGV.shift
-#
-#  # We want to create an async interface
-#  when /-async/
-#    $async = true
-#
-#  # We want to create an extension interface
-#  when /-extension/
-#    $extension = true
-#  end
-#
-#  # Go through the remaining command line arguments
-#  ARGV.each do |file|
-#
-#    # Check to see if an async interface file was requested
-#    if $async
-#
-#      AsyncInterface.print_out(file)
-#
-#    # Check if this is an extension file
-#    elsif $extension
-#      # Parse the extension file
-#      Extension.new(file)
-#
-#      # Print out the data
-#      ExtenionFile.print_out(file)
-#    end
-#
-#  end
-
-
-end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: scrapin-a-livin
 version: !ruby/object:Gem::Version 
-  version: 0.1.0
+  version: 0.1.1
 platform: ruby
 authors: 
 - Kevin S Kirkup
@@ -59,6 +59,7 @@ files:
 - README.markdown
 - Rakefile
 - VERSION
+- lib/dice/dice.rb
 - lib/dice/dice_search.rb
 - lib/generic/listing.rb
 - lib/scrapin-a-livin.rb