scrapin-a-livin 0.1.2 → 0.1.3
Sign up to get free protection for your applications and to get access to all the features.
- data/VERSION +1 -1
- data/lib/dice/dice.rb +1 -1
- data/lib/generic/listing.rb +18 -6
- data/lib/yahoo/hotjobs.rb +160 -159
- data/test/dice_parser_test.rb +162 -162
- data/test/listing_test.rb +13 -5
- data/test/yahoo_parser_test.rb +160 -160
- metadata +2 -3
- data/lib/dice/dice_search.rb +0 -122
data/test/listing_test.rb
CHANGED
|
@@ -20,7 +20,8 @@ class TC_JobListing < Test::Unit::TestCase
|
|
|
20
20
|
"Google Inc.",
|
|
21
21
|
"http://careers.google.com",
|
|
22
22
|
"Mountain View, CA",
|
|
23
|
-
"Dec, 1"
|
|
23
|
+
"Dec, 1",
|
|
24
|
+
"Dec, 5")
|
|
24
25
|
end
|
|
25
26
|
|
|
26
27
|
should "have same name" do
|
|
@@ -39,8 +40,8 @@ class TC_JobListing < Test::Unit::TestCase
|
|
|
39
40
|
assert_equal("http://careers.google.com", @listing.company_link)
|
|
40
41
|
end
|
|
41
42
|
|
|
42
|
-
should "have
|
|
43
|
-
assert_equal("Dec,
|
|
43
|
+
should "have reposted date" do
|
|
44
|
+
assert_equal("Dec, 5", @listing.date)
|
|
44
45
|
end
|
|
45
46
|
|
|
46
47
|
should "have parsed city" do
|
|
@@ -58,7 +59,8 @@ class TC_JobListing < Test::Unit::TestCase
|
|
|
58
59
|
"Company: Google Inc.\n" +
|
|
59
60
|
"Company Link: http://careers.google.com\n" +
|
|
60
61
|
"Location: Mountain View, CA\n" +
|
|
61
|
-
"Date: Dec, 1\n"
|
|
62
|
+
"Date: Dec, 1\n" +
|
|
63
|
+
"Repost: Dec, 5\n"
|
|
62
64
|
|
|
63
65
|
assert_equal(text, @listing.to_s)
|
|
64
66
|
end
|
|
@@ -76,7 +78,8 @@ class TC_JobListing < Test::Unit::TestCase
|
|
|
76
78
|
"Begin !@\#$%^&*()'~`;:'\"<>,./?&' End",
|
|
77
79
|
"http://careers.google.com",
|
|
78
80
|
"Begin !@\#$%^&*()'~`;:'\"<>./?&' End, CA",
|
|
79
|
-
"Dec, 1"
|
|
81
|
+
"Dec, 1",
|
|
82
|
+
nil)
|
|
80
83
|
|
|
81
84
|
# Create the temp directory
|
|
82
85
|
Dir.mkdir TMP_DIR
|
|
@@ -109,5 +112,10 @@ class TC_JobListing < Test::Unit::TestCase
|
|
|
109
112
|
assert_equal(path, @listing.add_city(TMP_DIR))
|
|
110
113
|
assert(true, File.exists?(path))
|
|
111
114
|
end
|
|
115
|
+
|
|
116
|
+
should "return the original date" do
|
|
117
|
+
assert_equal("Dec, 1", @listing.date)
|
|
118
|
+
end
|
|
119
|
+
|
|
112
120
|
end
|
|
113
121
|
end
|
data/test/yahoo_parser_test.rb
CHANGED
|
@@ -4,165 +4,165 @@ require 'ftools'
|
|
|
4
4
|
# Test Case for the yahoo hotjobs listing parser Dice
|
|
5
5
|
class TC_HotjobsSearchParser < Test::Unit::TestCase
|
|
6
6
|
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
7
|
+
RALEIGH_QUERY = File.dirname(__FILE__) + '/queries/hotjobs/queryRaleigh.html'
|
|
8
|
+
AUSTIN_QUERY = File.dirname(__FILE__) + '/queries/hotjobs/queryAustin.html'
|
|
9
|
+
SANJOSE_QUERY = File.dirname(__FILE__) + '/queries/hotjobs/querySanJose.html'
|
|
10
|
+
|
|
11
|
+
# Test for the listing header
|
|
12
|
+
def test_raleigh
|
|
13
|
+
|
|
14
|
+
# Read the query data from the file
|
|
15
|
+
data = check_query(RALEIGH_QUERY)
|
|
16
|
+
|
|
17
|
+
# Parse the listings from the query
|
|
18
|
+
listings = HotjobsSearch.parse_listings(data)
|
|
19
|
+
|
|
20
|
+
# Go through each listing
|
|
21
|
+
assert_equal("Embedded Software Engineer", listings[0].title)
|
|
22
|
+
assert_equal("http://hotjobs.yahoo.com/job-JNEJ91O9CZA?source=SRP", listings[0].link)
|
|
23
|
+
assert_equal("Sirit Technology", listings[0].company)
|
|
24
|
+
assert_equal("http://hotjobs.yahoo.com/careers-786030-Sirit_Technology", listings[0].company_link)
|
|
25
|
+
assert_equal("Morrisville, NC", listings[0].location)
|
|
26
|
+
assert_equal("Oct 28", listings[0].date)
|
|
27
|
+
|
|
28
|
+
assert_equal("MES Java Software Engineer", listings[1].title)
|
|
29
|
+
assert_equal("http://hotjobs.yahoo.com/job-JDZWR4DRM00?source=SRP", listings[1].link)
|
|
30
|
+
assert_equal("Werum Software & Systems America", listings[1].company)
|
|
31
|
+
assert_equal("http://hotjobs.yahoo.com/careers-616438-Werum_Software___Systems_America", listings[1].company_link)
|
|
32
|
+
assert_equal("Cary, NC", listings[1].location)
|
|
33
|
+
assert_equal("Nov 10", listings[1].date)
|
|
34
|
+
|
|
35
|
+
assert_equal("Software Developer", listings[2].title)
|
|
36
|
+
assert_equal("http://hotjobs.yahoo.com/job-JQGSVB1Y99F?source=SRP", listings[2].link)
|
|
37
|
+
assert_equal("CTG", listings[2].company)
|
|
38
|
+
assert_equal("http://hotjobs.yahoo.com/careers-601569-CTG", listings[2].company_link)
|
|
39
|
+
assert_equal("Durham, NC", listings[2].location)
|
|
40
|
+
assert_equal("Oct 30", listings[2].date)
|
|
41
|
+
|
|
42
|
+
assert_equal("Software Engineer - Windows Installer Developer, MSI, WiX, WiM", listings[3].title)
|
|
43
|
+
assert_equal("http://hotjobs.yahoo.com/job-J9WH2JP6WO7?source=SRP", listings[3].link)
|
|
44
|
+
assert_equal("CyberCoders", listings[3].company)
|
|
45
|
+
assert_equal("http://hotjobs.yahoo.com/careers-577525-CyberCoders", listings[3].company_link)
|
|
46
|
+
assert_equal("Raleigh, NC", listings[3].location)
|
|
47
|
+
assert_equal("Nov 27", listings[3].date)
|
|
48
|
+
|
|
49
|
+
assert_equal("C# Developer, Software Developer, SAAS, ASP.Net, SQL Server, SQL, .Net, WCF, JavaScript", listings[4].title)
|
|
50
|
+
assert_equal("http://hotjobs.yahoo.com/job-J5PKCPM1QM2?source=SRP", listings[4].link)
|
|
51
|
+
assert_equal("CyberCoders", listings[4].company)
|
|
52
|
+
assert_equal("http://hotjobs.yahoo.com/careers-577525-CyberCoders", listings[4].company_link)
|
|
53
|
+
assert_equal("Raleigh, NC", listings[4].location)
|
|
54
|
+
assert_equal("Nov 27", listings[4].date)
|
|
55
|
+
end
|
|
56
|
+
|
|
57
|
+
# Test for parsing a listing
|
|
58
|
+
def test_austin
|
|
59
|
+
|
|
60
|
+
# Read the query data
|
|
61
|
+
data = check_query(AUSTIN_QUERY)
|
|
62
|
+
|
|
63
|
+
# Parse the listings from the query
|
|
64
|
+
listings = HotjobsSearch.parse_listings(data)
|
|
65
|
+
|
|
66
|
+
# Go through each listing
|
|
67
|
+
assert_equal("Software Program Manager", listings[0].title)
|
|
68
|
+
assert_equal("http://hotjobs.yahoo.com/job-JSFNSIFFNCA?source=SRP", listings[0].link)
|
|
69
|
+
assert_equal("Zebra Imaging", listings[0].company)
|
|
70
|
+
assert_equal("http://hotjobs.yahoo.com/careers-627739-Zebra_Imaging", listings[0].company_link)
|
|
71
|
+
assert_equal("Austin, TX", listings[0].location)
|
|
72
|
+
assert_equal("Nov 19", listings[0].date)
|
|
73
|
+
|
|
74
|
+
assert_equal("Client Project Manager - Software", listings[1].title)
|
|
75
|
+
assert_equal("http://hotjobs.yahoo.com/job-JM1F32JSA7R?source=SRP", listings[1].link)
|
|
76
|
+
assert_equal("Digital Cheetah Solutions, Inc.", listings[1].company)
|
|
77
|
+
assert_equal("http://hotjobs.yahoo.com/careers-577804-Digital_Cheetah_Solutions__Inc_", listings[1].company_link)
|
|
78
|
+
assert_equal("Austin, TX", listings[1].location)
|
|
79
|
+
assert_equal("Nov 08", listings[1].date)
|
|
80
|
+
|
|
81
|
+
assert_equal("Software Quality Assurance Lead", listings[2].title)
|
|
82
|
+
assert_equal("http://hotjobs.yahoo.com/job-JL5SOECPQKF?source=SRP", listings[2].link)
|
|
83
|
+
assert_equal("Zebra Imaging", listings[2].company)
|
|
84
|
+
assert_equal("http://hotjobs.yahoo.com/careers-627739-Zebra_Imaging", listings[2].company_link)
|
|
85
|
+
assert_equal("Austin, TX", listings[2].location)
|
|
86
|
+
assert_equal("Oct 28", listings[2].date)
|
|
87
|
+
|
|
88
|
+
assert_equal("Operating System Software Test Specialist", listings[3].title)
|
|
89
|
+
assert_equal("http://hotjobs.yahoo.com/job-J3KBF8EADAO?source=SRP", listings[3].link)
|
|
90
|
+
assert_equal("CTG", listings[3].company)
|
|
91
|
+
assert_equal("http://hotjobs.yahoo.com/careers-601569-CTG", listings[3].company_link)
|
|
92
|
+
assert_equal("Austin, TX", listings[3].location)
|
|
93
|
+
assert_equal("Nov 12", listings[3].date)
|
|
94
|
+
|
|
95
|
+
assert_equal("Software Engineer", listings[4].title)
|
|
96
|
+
assert_equal("http://hotjobs.yahoo.com/job-JTXD1K5GAZ2?source=SRP", listings[4].link)
|
|
97
|
+
assert_equal("Troux Technologies", listings[4].company)
|
|
98
|
+
assert_equal("http://hotjobs.yahoo.com/careers-788124-Troux_Technologies", listings[4].company_link)
|
|
99
|
+
assert_equal("Austin, TX", listings[4].location)
|
|
100
|
+
assert_equal("Nov 05", listings[4].date)
|
|
101
|
+
|
|
102
|
+
end
|
|
103
|
+
|
|
104
|
+
# Test for parsing a listing
|
|
105
|
+
def test_sanjose
|
|
106
|
+
|
|
107
|
+
# Read the query data
|
|
108
|
+
data = check_query(SANJOSE_QUERY)
|
|
109
|
+
|
|
110
|
+
# Parse the listings from the query
|
|
111
|
+
listings = HotjobsSearch.parse_listings(data)
|
|
112
|
+
|
|
113
|
+
# Go through each listing
|
|
114
|
+
assert_equal("Software Engineer/Carrier CE", listings[0].title)
|
|
115
|
+
assert_equal("http://hotjobs.yahoo.com/job-J2VW8Y7GMAH?source=SRP", listings[0].link)
|
|
116
|
+
assert_equal("Atheros Communications, Inc.", listings[0].company)
|
|
117
|
+
assert_equal("http://hotjobs.yahoo.com/careers-552561-Atheros_Communications__Inc_", listings[0].company_link)
|
|
118
|
+
assert_equal("Santa Clara, CA", listings[0].location)
|
|
119
|
+
assert_equal("Nov 12", listings[0].date)
|
|
120
|
+
|
|
121
|
+
assert_equal("EMBEDDED SOFTWARE ENGINEER", listings[1].title)
|
|
122
|
+
assert_equal("http://hotjobs.yahoo.com/job-JD285WTGJLP?source=SRP", listings[1].link)
|
|
123
|
+
assert_equal("NVIDIA Corporation", listings[1].company)
|
|
124
|
+
assert_equal("http://hotjobs.yahoo.com/careers-601859-NVIDIA_Corporation", listings[1].company_link)
|
|
125
|
+
assert_equal("Santa Clara, CA", listings[1].location)
|
|
126
|
+
assert_equal("Nov 25", listings[1].date)
|
|
127
|
+
|
|
128
|
+
assert_equal("EMBEDDED SOFTWARE ENGINEER", listings[2].title)
|
|
129
|
+
assert_equal("http://hotjobs.yahoo.com/job-J9LJP1J38EB?source=SRP", listings[2].link)
|
|
130
|
+
assert_equal("NVIDIA Corporation", listings[2].company)
|
|
131
|
+
assert_equal("http://hotjobs.yahoo.com/careers-601859-NVIDIA_Corporation", listings[2].company_link)
|
|
132
|
+
assert_equal("Santa Clara, CA", listings[2].location)
|
|
133
|
+
assert_equal("Nov 25", listings[2].date)
|
|
134
|
+
|
|
135
|
+
assert_equal("Platform Bringup Software Engineer", listings[3].title)
|
|
136
|
+
assert_equal("http://hotjobs.yahoo.com/job-J6YYFVVFK7P?source=SRP", listings[3].link)
|
|
137
|
+
assert_equal("Aruba Networks", listings[3].company)
|
|
138
|
+
assert_equal("http://hotjobs.yahoo.com/careers-562030-Aruba_Networks", listings[3].company_link)
|
|
139
|
+
assert_equal("Sunnyvale, CA", listings[3].location)
|
|
140
|
+
assert_equal("Nov 21", listings[3].date)
|
|
141
|
+
|
|
142
|
+
assert_equal("Staff Software Development Engineer", listings[4].title)
|
|
143
|
+
assert_equal("http://hotjobs.yahoo.com/job-JDG4ZD5RP59?source=SRP", listings[4].link)
|
|
144
|
+
assert_equal("IDT", listings[4].company)
|
|
145
|
+
assert_equal("http://hotjobs.yahoo.com/careers-577851-IDT", listings[4].company_link)
|
|
146
|
+
assert_equal("San Jose, CA", listings[4].location)
|
|
147
|
+
assert_equal("Nov 25", listings[4].date)
|
|
148
|
+
end
|
|
149
|
+
|
|
150
|
+
private
|
|
151
|
+
|
|
152
|
+
# Make sure the query file exists
|
|
153
|
+
#
|
|
154
|
+
# @param path [String, #read] the path to the input file
|
|
155
|
+
# @returns [String] the data from the input file
|
|
156
|
+
def check_query(path)
|
|
157
|
+
|
|
158
|
+
# Check to make sure the query exists and is readable
|
|
159
|
+
if !File.exists?(path) || !File.readable?(path)
|
|
160
|
+
|
|
161
|
+
raise ArgumentError.new "Invalid Query: \"#{path}\""
|
|
162
|
+
end
|
|
163
|
+
|
|
164
|
+
# Open the file and read the contents
|
|
165
|
+
File.open(path, "r").read
|
|
166
|
+
end
|
|
167
167
|
|
|
168
168
|
end
|
metadata
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: scrapin-a-livin
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.1.
|
|
4
|
+
version: 0.1.3
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Kevin S Kirkup
|
|
@@ -9,7 +9,7 @@ autorequire:
|
|
|
9
9
|
bindir: bin
|
|
10
10
|
cert_chain: []
|
|
11
11
|
|
|
12
|
-
date: 2009-
|
|
12
|
+
date: 2009-12-01 00:00:00 -05:00
|
|
13
13
|
default_executable:
|
|
14
14
|
dependencies:
|
|
15
15
|
- !ruby/object:Gem::Dependency
|
|
@@ -60,7 +60,6 @@ files:
|
|
|
60
60
|
- Rakefile
|
|
61
61
|
- VERSION
|
|
62
62
|
- lib/dice/dice.rb
|
|
63
|
-
- lib/dice/dice_search.rb
|
|
64
63
|
- lib/generic/listing.rb
|
|
65
64
|
- lib/scrapin-a-livin.rb
|
|
66
65
|
- lib/yahoo/hotjobs.rb
|
data/lib/dice/dice_search.rb
DELETED
|
@@ -1,122 +0,0 @@
|
|
|
1
|
-
#!/usr/bin/env ruby
|
|
2
|
-
require 'rubygems'
|
|
3
|
-
require 'open-uri'
|
|
4
|
-
require 'hpricot'
|
|
5
|
-
|
|
6
|
-
# Class to screen scrape the dice website
|
|
7
|
-
#
|
|
8
|
-
# http://seeker.dice.com
|
|
9
|
-
class DiceSearch
|
|
10
|
-
|
|
11
|
-
# Constants
|
|
12
|
-
DEBUG = false
|
|
13
|
-
TITLE_CELL = 2
|
|
14
|
-
COMPANY_CELL = 3
|
|
15
|
-
LOCATION_CELL = 4
|
|
16
|
-
DATE_CELL = 5
|
|
17
|
-
CELL_COUNT = 6
|
|
18
|
-
DICE_LINK = "http://seeker.dice.com"
|
|
19
|
-
|
|
20
|
-
# Parse the provided query data
|
|
21
|
-
#
|
|
22
|
-
# @param query [String, #read] the html web page data
|
|
23
|
-
# @returns [JobListings] an array of job listings
|
|
24
|
-
def self.parse_listings(query)
|
|
25
|
-
|
|
26
|
-
# Create the listings
|
|
27
|
-
listings = Array.new
|
|
28
|
-
|
|
29
|
-
# Filter the data with Hpricot
|
|
30
|
-
doc = Hpricot(query)
|
|
31
|
-
|
|
32
|
-
# Get the table
|
|
33
|
-
table = (doc/"//table[@class=summary]")
|
|
34
|
-
|
|
35
|
-
# Get the rows
|
|
36
|
-
rows = (table/"tr")
|
|
37
|
-
|
|
38
|
-
# Retrieve the table rows that contain the job listings
|
|
39
|
-
rows.each { |row|
|
|
40
|
-
|
|
41
|
-
# Get the individual cells
|
|
42
|
-
cells = (row/"td")
|
|
43
|
-
|
|
44
|
-
# If this is a job listing
|
|
45
|
-
if cells.size == CELL_COUNT
|
|
46
|
-
|
|
47
|
-
# Get the fields
|
|
48
|
-
name = (cells[TITLE_CELL]/"a").inner_html
|
|
49
|
-
link = DICE_LINK + (cells[TITLE_CELL]/"a").attr("href")
|
|
50
|
-
company = (cells[COMPANY_CELL]/"a").inner_html
|
|
51
|
-
company_link = DICE_LINK + (cells[COMPANY_CELL]/"a").attr("href")
|
|
52
|
-
location = cells[LOCATION_CELL].inner_html
|
|
53
|
-
date = cells[DATE_CELL].inner_html
|
|
54
|
-
|
|
55
|
-
if DEBUG
|
|
56
|
-
puts "Row: count #{cells.size}"
|
|
57
|
-
puts "Name: #{name}"
|
|
58
|
-
puts "Link: #{link}"
|
|
59
|
-
puts "Company: #{company}"
|
|
60
|
-
puts "Company Link: #{company_link}"
|
|
61
|
-
puts "Location: #{location}"
|
|
62
|
-
puts "Date: #{date}"
|
|
63
|
-
end
|
|
64
|
-
|
|
65
|
-
# Create the job listing
|
|
66
|
-
listings << JobListing.new(name, link, company, company_link, location, date)
|
|
67
|
-
|
|
68
|
-
end
|
|
69
|
-
|
|
70
|
-
}
|
|
71
|
-
|
|
72
|
-
# Return the listings
|
|
73
|
-
return listings
|
|
74
|
-
|
|
75
|
-
end
|
|
76
|
-
|
|
77
|
-
# Retrieve the job listings
|
|
78
|
-
#
|
|
79
|
-
# @param url [String, #read] the url used to query the data
|
|
80
|
-
# @param [JobListings] an array of job listings
|
|
81
|
-
def self.get_listings(url)
|
|
82
|
-
|
|
83
|
-
# Read the data from the url
|
|
84
|
-
response = open(url, "User-Agent" => "Ruby/#{RUBY_VERSION}",
|
|
85
|
-
"From" => "email@addr.com",
|
|
86
|
-
"Referer" => "http://hotjobs.yahoo.com/").read
|
|
87
|
-
|
|
88
|
-
# Parse the listings from the query
|
|
89
|
-
parse_listings(response)
|
|
90
|
-
|
|
91
|
-
end
|
|
92
|
-
|
|
93
|
-
# Query dice for html code for the query
|
|
94
|
-
#
|
|
95
|
-
# @param location [String, #read] the location to search
|
|
96
|
-
# @param keywords [String, #read] keywords to use for the search
|
|
97
|
-
# @param days_back [String, #read] how long ago to search
|
|
98
|
-
# @param num_entries [String, #read] the number of entries to request
|
|
99
|
-
def self.query(location, keywords, days_back, num_entries)
|
|
100
|
-
|
|
101
|
-
# The search URL
|
|
102
|
-
url = "http://seeker.dice.com/jobsearch/servlet/JobSearch" +
|
|
103
|
-
"?caller=0&LOCATION_OPTION=2&EXTRA_STUFF=1&N=0&Hf=0" +
|
|
104
|
-
"&Ntk=JobSearchRanking&op=300" +
|
|
105
|
-
"&values=&FREE_TEXT=#{keywords}" +
|
|
106
|
-
"&Ntx=mode+matchall&WHERE=#{location}" +
|
|
107
|
-
"&WHEREList=#{location}" +
|
|
108
|
-
"&RADIUS=80.4672" +
|
|
109
|
-
"&COUNTRY=1525&STAT_PROV=0&METRO_AREA=33.78715899%2C-84.39164034&AREA_CODES=&AC_COUNTRY=1525" +
|
|
110
|
-
"&TRAVEL=0&TAXTERM=1001" +
|
|
111
|
-
"&SORTSPEC=0" +
|
|
112
|
-
"&FRMT=0" +
|
|
113
|
-
"&DAYSBACK=#{days_back}" +
|
|
114
|
-
"&NUM_PER_PAGE=#{num_entries}"
|
|
115
|
-
|
|
116
|
-
# Read the data from the url
|
|
117
|
-
open(@url, "User-Agent" => "Ruby/#{RUBY_VERSION}",
|
|
118
|
-
"From" => "email@addr.com",
|
|
119
|
-
"Referer" => "http://seeker.dice.com/jobsearch/").read
|
|
120
|
-
end
|
|
121
|
-
|
|
122
|
-
end
|