rubyscraper 0.3.0 → 0.9.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (20) hide show
  1. checksums.yaml +4 -4
  2. data/Gemfile.lock +5 -6
  3. data/README.md +37 -7
  4. data/lib/rubyscraper.rb +14 -148
  5. data/lib/rubyscraper/api_dispatcher.rb +31 -0
  6. data/lib/rubyscraper/binary.rb +9 -6
  7. data/lib/rubyscraper/option_parser.rb +72 -0
  8. data/lib/rubyscraper/paginator.rb +59 -0
  9. data/lib/rubyscraper/processor.rb +47 -0
  10. data/lib/rubyscraper/sub_page_scraper.rb +53 -0
  11. data/lib/rubyscraper/summary_scraper.rb +65 -0
  12. data/lib/rubyscraper/version.rb +1 -1
  13. data/rubyscraper.gemspec +5 -6
  14. data/spec/paginator_spec.rb +83 -0
  15. data/spec/rubyscraper_spec.rb +2 -6
  16. data/spec/spec_helper.rb +3 -0
  17. data/spec/sub_page_scraper_spec.rb +51 -0
  18. data/spec/summary_scraper_spec.rb +125 -0
  19. metadata +27 -33
  20. data/lib/assets/scrapes.json +0 -287
metadata CHANGED
@@ -1,71 +1,57 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: rubyscraper
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.3.0
4
+ version: 0.9.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Nathan Owsiany
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2015-04-26 00:00:00.000000000 Z
11
+ date: 2015-04-28 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: capybara
15
15
  requirement: !ruby/object:Gem::Requirement
16
16
  requirements:
17
- - - ">="
17
+ - - "~>"
18
18
  - !ruby/object:Gem::Version
19
- version: '0'
19
+ version: '2.4'
20
20
  type: :runtime
21
21
  prerelease: false
22
22
  version_requirements: !ruby/object:Gem::Requirement
23
23
  requirements:
24
- - - ">="
24
+ - - "~>"
25
25
  - !ruby/object:Gem::Version
26
- version: '0'
26
+ version: '2.4'
27
27
  - !ruby/object:Gem::Dependency
28
28
  name: poltergeist
29
29
  requirement: !ruby/object:Gem::Requirement
30
30
  requirements:
31
- - - ">="
31
+ - - "~>"
32
32
  - !ruby/object:Gem::Version
33
- version: '0'
33
+ version: '1.6'
34
34
  type: :runtime
35
35
  prerelease: false
36
36
  version_requirements: !ruby/object:Gem::Requirement
37
37
  requirements:
38
- - - ">="
38
+ - - "~>"
39
39
  - !ruby/object:Gem::Version
40
- version: '0'
40
+ version: '1.6'
41
41
  - !ruby/object:Gem::Dependency
42
42
  name: rest-client
43
43
  requirement: !ruby/object:Gem::Requirement
44
44
  requirements:
45
- - - ">="
46
- - !ruby/object:Gem::Version
47
- version: '0'
48
- type: :runtime
49
- prerelease: false
50
- version_requirements: !ruby/object:Gem::Requirement
51
- requirements:
52
- - - ">="
53
- - !ruby/object:Gem::Version
54
- version: '0'
55
- - !ruby/object:Gem::Dependency
56
- name: slop
57
- requirement: !ruby/object:Gem::Requirement
58
- requirements:
59
- - - ">="
45
+ - - "~>"
60
46
  - !ruby/object:Gem::Version
61
- version: '0'
47
+ version: '1.8'
62
48
  type: :runtime
63
49
  prerelease: false
64
50
  version_requirements: !ruby/object:Gem::Requirement
65
51
  requirements:
66
- - - ">="
52
+ - - "~>"
67
53
  - !ruby/object:Gem::Version
68
- version: '0'
54
+ version: '1.8'
69
55
  - !ruby/object:Gem::Dependency
70
56
  name: bundler
71
57
  requirement: !ruby/object:Gem::Requirement
@@ -112,16 +98,16 @@ dependencies:
112
98
  name: pry
113
99
  requirement: !ruby/object:Gem::Requirement
114
100
  requirements:
115
- - - ">="
101
+ - - "~>"
116
102
  - !ruby/object:Gem::Version
117
- version: '0'
103
+ version: '0.10'
118
104
  type: :development
119
105
  prerelease: false
120
106
  version_requirements: !ruby/object:Gem::Requirement
121
107
  requirements:
122
- - - ">="
108
+ - - "~>"
123
109
  - !ruby/object:Gem::Version
124
- version: '0'
110
+ version: '0.10'
125
111
  description: Scrapes job sites for job details and sends post request to server.
126
112
  email: nowsiany@gmail.com
127
113
  executables:
@@ -136,13 +122,21 @@ files:
136
122
  - bin/console
137
123
  - bin/rubyscraper
138
124
  - bin/setup
139
- - lib/assets/scrapes.json
140
125
  - lib/rubyscraper.rb
126
+ - lib/rubyscraper/api_dispatcher.rb
141
127
  - lib/rubyscraper/binary.rb
128
+ - lib/rubyscraper/option_parser.rb
129
+ - lib/rubyscraper/paginator.rb
130
+ - lib/rubyscraper/processor.rb
131
+ - lib/rubyscraper/sub_page_scraper.rb
132
+ - lib/rubyscraper/summary_scraper.rb
142
133
  - lib/rubyscraper/version.rb
143
134
  - rubyscraper.gemspec
135
+ - spec/paginator_spec.rb
144
136
  - spec/rubyscraper_spec.rb
145
137
  - spec/spec_helper.rb
138
+ - spec/sub_page_scraper_spec.rb
139
+ - spec/summary_scraper_spec.rb
146
140
  homepage: https://github.com/ndwhtlssthr/rubyscraper
147
141
  licenses:
148
142
  - MIT
data/lib/assets/scrapes.json DELETED
@@ -1,287 +0,0 @@
1
- [
2
- {
3
- "name":"stackoverflow",
4
- "base_url":"http://www.careers.stackoverflow.com",
5
- "summary":{
6
- "url":"/jobs?searchTerm=SEARCHTERM&sort=p",
7
- "pagination_fmt":"&pg=",
8
- "pagination_start":"1",
9
- "pagination_scale":"1",
10
- "params":[
11
- {
12
- "SEARCHTERM":[
13
- "ruby",
14
- "ruby+on+rails",
15
- "javascript"
16
- ]
17
- }
18
- ],
19
- "loop":".listResults .-item",
20
- "fields":[
21
- {
22
- "field":"position",
23
- "method":"find",
24
- "path":"h3.-title a"
25
- },
26
- {
27
- "field":"url",
28
- "method":"find",
29
- "path":"h3.-title a",
30
- "attr":"href"
31
- },
32
- {
33
- "field":"posting_date",
34
- "method":"first",
35
- "path":"p._muted"
36
- }
37
- ]
38
- },
39
- "sub_page":{
40
- "fields":[
41
- {
42
- "field":"company",
43
- "method":"find",
44
- "path":"a.employer"
45
- },
46
- {
47
- "field":"location",
48
- "method":"find",
49
- "path":"span.location"
50
- },
51
- {
52
- "field":"description",
53
- "method":"all",
54
- "path":"div.description p",
55
- "loop_collect":"text",
56
- "join":"\n"
57
- },
58
- {
59
- "field":"tags",
60
- "method":"all",
61
- "path":"div.tags a.post-tag",
62
- "loop_collect":"text",
63
- "join":", "
64
- }
65
- ]
66
- }
67
- },
68
- {
69
- "name":"rubynow",
70
- "base_url":"http://jobs.rubynow.com/",
71
- "summary":{
72
- "url":"",
73
- "no_pagination?":"true",
74
- "pagination_fmt":"",
75
- "pagination_start":"",
76
- "pagination_scale":"",
77
- "params":[
78
- ],
79
- "loop":"ul.jobs li",
80
- "fields":[
81
- {
82
- "field":"position",
83
- "method":"find",
84
- "path":"h3 a"
85
- },
86
- {
87
- "field":"url",
88
- "method":"find",
89
- "path":"h3 a",
90
- "attr":"href"
91
- },
92
- {
93
- "field":"posting_date",
94
- "method":"find",
95
- "path":"span.date"
96
- }
97
- ]
98
- },
99
- "sub_page":{
100
- "fields":[
101
- {
102
- "field":"description",
103
- "method":"all",
104
- "path":"div#info p",
105
- "loop_collect":"text",
106
- "join":"\n"
107
- },
108
- {
109
- "field":"company",
110
- "method":"find",
111
- "path":"h2#headline a"
112
- },
113
- {
114
- "field":"location",
115
- "method":"find",
116
- "path":"h3#location"
117
- }
118
- ]
119
- }
120
- },
121
- {
122
- "name":"weworkremotely",
123
- "base_url":"https://weworkremotely.com",
124
- "summary":{
125
- "url":"/categories/2/jobs",
126
- "no_pagination?":"true",
127
- "pagination_fmt":"",
128
- "pagination_start":"",
129
- "pagination_scale":"",
130
- "params":[
131
- ],
132
- "loop":"section.jobs ul li",
133
- "fields":[
134
- {
135
- "field":"position",
136
- "method":"find",
137
- "path":"span.title"
138
- },
139
- {
140
- "field":"company",
141
- "method":"find",
142
- "path":"span.company"
143
- },
144
- {
145
- "field":"url",
146
- "method":"find",
147
- "path":"a",
148
- "attr":"href"
149
- },
150
- {
151
- "field":"posting_date",
152
- "method":"find",
153
- "path":"span.date"
154
- }
155
- ]
156
- },
157
- "sub_page":{
158
- "fields":[
159
- {
160
- "field":"location",
161
- "method":"find",
162
- "path":"span.location"
163
- },
164
- {
165
- "field":"description",
166
- "method":"all",
167
- "path":"div.listing-container div",
168
- "loop_collect":"text",
169
- "join":"\n"
170
- }
171
- ]
172
- }
173
- },
174
- {
175
- "name":"indeed",
176
- "skip":"true",
177
- "base_url":"http://www.indeed.com",
178
- "summary":{
179
- "url":"/jobs?q=SEARCHTERM&sr=directhire",
180
- "pagination_fmt":"&start=",
181
- "pagination_start":"0",
182
- "pagination_scale":"10",
183
- "params":[
184
- {
185
- "SEARCHTERM":[
186
- "ruby",
187
- "ruby+on+rails",
188
- "junior+web+developer",
189
- "ember.js",
190
- "full+stack"
191
- ]
192
- }
193
- ],
194
- "loop":"div.row.result",
195
- "fields":[
196
- {
197
- "field":"position",
198
- "method":"find",
199
- "path":"h2.jobtitle a"
200
- },
201
- {
202
- "field":"url",
203
- "method":"find",
204
- "path":"h2.jobtitle a",
205
- "attr":"href"
206
- },
207
- {
208
- "field":"company",
209
- "method":"find",
210
- "path":"span.company span"
211
- },
212
- {
213
- "field":"location",
214
- "method":"find",
215
- "path":"span.location span"
216
- },
217
- {
218
- "field":"description",
219
- "method":"find",
220
- "path":"span.summary span"
221
- },
222
- {
223
- "field":"posting_date",
224
- "method":"find",
225
- "path":"span.date"
226
- }
227
- ]
228
- },
229
- "sub_page":{
230
- "fields":[
231
- ]
232
- }
233
- },
234
- {
235
- "name":"linkedin",
236
- "skip":"true",
237
- "base_url":"https://www.linkedin.com",
238
- "summary":{
239
- "url":"/vsearch/j?keywords=SEARCHTERM&openAdvancedForm=true&locationType=I&countryCode=us&rsid=754744171429892349899&orig=FCTD&openFacets=L,C,TP&f_TP=1&pt=jobs&pt=jobs",
240
- "pagination_fmt":"&page_num=",
241
- "pagination_start":"1",
242
- "pagination_scale":"1",
243
- "params":[
244
- {
245
- "SEARCHTERM":[
246
- "Ruby",
247
- "Ruby+On+Rails",
248
- "javascript"
249
- ]
250
- }
251
- ],
252
- "loop":"ol.search-results li.result",
253
- "fields":[
254
- {
255
- "field":"position",
256
- "method":"find",
257
- "path":"a.title"
258
- },
259
- {
260
- "field":"url",
261
- "method":"find",
262
- "path":"a.title",
263
- "attr":"href"
264
- },
265
- {
266
- "field":"company",
267
- "method":"find",
268
- "path":"div.description a"
269
- },
270
- {
271
- "field":"location",
272
- "method":"find",
273
- "path":"dl.demographic bdi"
274
- }
275
- ]
276
- },
277
- "sub_page":{
278
- "fields":[
279
- {
280
- "field":"description",
281
- "method":"find",
282
- "path":"div.description-section div.rich-text"
283
- }
284
- ]
285
- }
286
- }
287
- ]