osu-cc-scraper 1.0.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/bin/osu-cc-scraper +12 -0
- data/lib/osu-cc-scraper/course.rb +37 -0
- data/lib/osu-cc-scraper/department.rb +40 -0
- data/lib/osu-cc-scraper/section.rb +64 -0
- data/lib/osu-cc-scraper.rb +7 -0
- metadata +63 -0
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA1:
|
3
|
+
metadata.gz: f3a4b341ef148c517c8e0bce60cd66b5626fb5bb
|
4
|
+
data.tar.gz: be876d5f07610468ae75ea6c96d864faeeb42cef
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: b33d730a9e4dccbe85477dbbd02137cdbd23feea22b67d1355eb6003d3ab066fe7da2d46a9a1a6fa94f878dc696595ea33146bac8d9ef7b296f460758d6cd728
|
7
|
+
data.tar.gz: 8073fa241a16ffd2553e0e9a5049c394fef00ddf5c2b8aa7091856e00a04ca0ffb6d500a47deec7439c2208a6a8e932fb1bd3148ac8156584997595bdcbcb38d
|
data/bin/osu-cc-scraper
ADDED
@@ -0,0 +1,37 @@
|
|
1
|
+
require "open-uri"
|
2
|
+
require "nokogiri"
|
3
|
+
require "osu-cc-scraper/section"
|
4
|
+
|
5
|
+
class OsuCcScraper::Course
|
6
|
+
attr_accessor :subject_code
|
7
|
+
attr_accessor :course_number
|
8
|
+
|
9
|
+
def initialize(args)
|
10
|
+
args.each do |k,v|
|
11
|
+
instance_variable_set("@#{k}", v) unless v.nil?
|
12
|
+
end
|
13
|
+
end
|
14
|
+
|
15
|
+
def sections
|
16
|
+
html = OsuCcScraper::Section::fetch(self.subject_code, self.course_number)
|
17
|
+
OsuCcScraper::Section::parse(html)
|
18
|
+
end
|
19
|
+
|
20
|
+
private
|
21
|
+
|
22
|
+
def self.fetch(subject_code)
|
23
|
+
open("#{OsuCcScraper::ENDPOINT}/CourseList.aspx?subjectcode=#{subject_code}&level=undergrad&campus=corvallis").read
|
24
|
+
end
|
25
|
+
|
26
|
+
def self.parse(html)
|
27
|
+
ng = Nokogiri::HTML(html)
|
28
|
+
ng.css("tr td strong a:last").map{ |course|
|
29
|
+
uri = course.attribute('href').to_s.gsub('CourseDetail.aspx?', '')
|
30
|
+
params = URI.decode_www_form(uri).to_h
|
31
|
+
OsuCcScraper::Course.new({
|
32
|
+
subject_code: params["subjectcode"],
|
33
|
+
course_number: params["coursenumber"]
|
34
|
+
})
|
35
|
+
}
|
36
|
+
end
|
37
|
+
end
|
@@ -0,0 +1,40 @@
|
|
1
|
+
require "open-uri"
|
2
|
+
require "nokogiri"
|
3
|
+
require "osu-cc-scraper/course"
|
4
|
+
|
5
|
+
class OsuCcScraper::Department
|
6
|
+
attr_accessor :subject_code
|
7
|
+
|
8
|
+
def initialize(args)
|
9
|
+
args.each do |k,v|
|
10
|
+
instance_variable_set("@#{k}", v) unless v.nil?
|
11
|
+
end
|
12
|
+
end
|
13
|
+
|
14
|
+
def self.all
|
15
|
+
html = OsuCcScraper::Department::fetch
|
16
|
+
OsuCcScraper::Department::parse(html)
|
17
|
+
end
|
18
|
+
|
19
|
+
def courses
|
20
|
+
html = OsuCcScraper::Course::fetch(self.subject_code)
|
21
|
+
OsuCcScraper::Course::parse(html)
|
22
|
+
end
|
23
|
+
|
24
|
+
private
|
25
|
+
|
26
|
+
def self.fetch
|
27
|
+
open("#{OsuCcScraper::ENDPOINT}/CourseDescription.aspx?level=undergrad").read
|
28
|
+
end
|
29
|
+
|
30
|
+
def self.parse(html)
|
31
|
+
ng = Nokogiri::HTML(html)
|
32
|
+
ng.xpath("//tr/td/font/a").map{ |department|
|
33
|
+
OsuCcScraper::Department.new({
|
34
|
+
subject_code: department.content[/\(.*?\)/][1..-2]
|
35
|
+
})
|
36
|
+
}.sort { |a,b|
|
37
|
+
a.subject_code.downcase <=> b.subject_code.downcase
|
38
|
+
}
|
39
|
+
end
|
40
|
+
end
|
@@ -0,0 +1,64 @@
|
|
1
|
+
require "open-uri"
|
2
|
+
require "nokogiri"
|
3
|
+
|
4
|
+
class OsuCcScraper::Section
|
5
|
+
attr_accessor :department
|
6
|
+
attr_accessor :number
|
7
|
+
attr_accessor :name
|
8
|
+
attr_accessor :term
|
9
|
+
attr_accessor :section
|
10
|
+
attr_accessor :instructor
|
11
|
+
attr_accessor :type
|
12
|
+
attr_accessor :status
|
13
|
+
attr_accessor :capacity
|
14
|
+
attr_accessor :availability
|
15
|
+
|
16
|
+
def initialize(args)
|
17
|
+
args.each do |k,v|
|
18
|
+
instance_variable_set("@#{k}", v) unless v.nil?
|
19
|
+
end
|
20
|
+
end
|
21
|
+
|
22
|
+
def to_a
|
23
|
+
[@department, @number, @name, @term, @section, @instructor, @type, @status, @capacity, @availability]
|
24
|
+
end
|
25
|
+
|
26
|
+
private
|
27
|
+
|
28
|
+
def self.fetch(subject_code, course_number)
|
29
|
+
open("#{OsuCcScraper::ENDPOINT}/CourseDetail.aspx?subjectcode=#{subject_code}&coursenumber=#{course_number}").read
|
30
|
+
end
|
31
|
+
|
32
|
+
def self.parse(html)
|
33
|
+
sections = []
|
34
|
+
|
35
|
+
ng = Nokogiri::HTML(html)
|
36
|
+
rows = ng.xpath('//table[@id="ctl00_ContentPlaceHolder1_SOCListUC1_gvOfferings"]/tr[position() > 1]')
|
37
|
+
rows.each_with_index do |row, key|
|
38
|
+
title = ng.at_xpath('//form/h3').content.split("\n")
|
39
|
+
query = "//table[@id='ctl00_ContentPlaceHolder1_SOCListUC1_gvOfferings']"
|
40
|
+
|
41
|
+
sections << OsuCcScraper::Section.new({
|
42
|
+
:department => title[2].delete("\r\n").strip[0..-2].split(" ")[0],
|
43
|
+
:number => title[2].split(" ")[1][0..-2],
|
44
|
+
:name => title[3].strip,
|
45
|
+
:term => at_xpath_or_null(ng, "#{query}/tr[position() = #{key+2}]/td[position() = 1]"),
|
46
|
+
:section => at_xpath_or_null(ng, "#{query}/tr[position() = #{key+2}]/td[position() = 3]"),
|
47
|
+
:instructor => at_xpath_or_null(ng, "#{query}/tr[position() = #{key+2}]/td[position() = 6]"),
|
48
|
+
:type => at_xpath_or_null(ng, "#{query}/tr[position() = #{key+2}]/td[position() = 11]"),
|
49
|
+
:status => at_xpath_or_null(ng, "#{query}/tr[position() = #{key+2}]/td[position() = 12]"),
|
50
|
+
:capacity => at_xpath_or_null(ng, "#{query}/tr[position() = #{key+2}]/td[position() = 13]"),
|
51
|
+
:availability => at_xpath_or_null(ng, "#{query}/tr[position() = #{key+2}]/td[position() = 14]")
|
52
|
+
})
|
53
|
+
end
|
54
|
+
sections
|
55
|
+
end
|
56
|
+
|
57
|
+
def self.at_xpath_or_null(document, selector)
|
58
|
+
if document.at_xpath(selector) != nil
|
59
|
+
return document.at_xpath(selector).content.delete("\r\n").strip
|
60
|
+
else
|
61
|
+
return nil
|
62
|
+
end
|
63
|
+
end
|
64
|
+
end
|
metadata
ADDED
@@ -0,0 +1,63 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: osu-cc-scraper
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 1.0.1
|
5
|
+
platform: ruby
|
6
|
+
authors:
|
7
|
+
- Jonah George
|
8
|
+
autorequire:
|
9
|
+
bindir: bin
|
10
|
+
cert_chain: []
|
11
|
+
date: 2015-08-10 00:00:00.000000000 Z
|
12
|
+
dependencies:
|
13
|
+
- !ruby/object:Gem::Dependency
|
14
|
+
name: nokogiri
|
15
|
+
requirement: !ruby/object:Gem::Requirement
|
16
|
+
requirements:
|
17
|
+
- - '='
|
18
|
+
- !ruby/object:Gem::Version
|
19
|
+
version: 1.6.6.2
|
20
|
+
type: :runtime
|
21
|
+
prerelease: false
|
22
|
+
version_requirements: !ruby/object:Gem::Requirement
|
23
|
+
requirements:
|
24
|
+
- - '='
|
25
|
+
- !ruby/object:Gem::Version
|
26
|
+
version: 1.6.6.2
|
27
|
+
description: A script to gather course data from Oregon State's Course Catalog.
|
28
|
+
email: jonah.george@me.com
|
29
|
+
executables:
|
30
|
+
- osu-cc-scraper
|
31
|
+
extensions: []
|
32
|
+
extra_rdoc_files: []
|
33
|
+
files:
|
34
|
+
- bin/osu-cc-scraper
|
35
|
+
- lib/osu-cc-scraper.rb
|
36
|
+
- lib/osu-cc-scraper/course.rb
|
37
|
+
- lib/osu-cc-scraper/department.rb
|
38
|
+
- lib/osu-cc-scraper/section.rb
|
39
|
+
homepage: https://github.com/jonahgeorge/osu-cc-scraper
|
40
|
+
licenses:
|
41
|
+
- MIT
|
42
|
+
metadata: {}
|
43
|
+
post_install_message:
|
44
|
+
rdoc_options: []
|
45
|
+
require_paths:
|
46
|
+
- lib
|
47
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
48
|
+
requirements:
|
49
|
+
- - ">="
|
50
|
+
- !ruby/object:Gem::Version
|
51
|
+
version: '0'
|
52
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
53
|
+
requirements:
|
54
|
+
- - ">="
|
55
|
+
- !ruby/object:Gem::Version
|
56
|
+
version: '0'
|
57
|
+
requirements: []
|
58
|
+
rubyforge_project:
|
59
|
+
rubygems_version: 2.4.8
|
60
|
+
signing_key:
|
61
|
+
specification_version: 4
|
62
|
+
summary: OSU Course Catalog Scraper
|
63
|
+
test_files: []
|