osu-cc-scraper 1.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: f3a4b341ef148c517c8e0bce60cd66b5626fb5bb
+  data.tar.gz: be876d5f07610468ae75ea6c96d864faeeb42cef
+SHA512:
+  metadata.gz: b33d730a9e4dccbe85477dbbd02137cdbd23feea22b67d1355eb6003d3ab066fe7da2d46a9a1a6fa94f878dc696595ea33146bac8d9ef7b296f460758d6cd728
+  data.tar.gz: 8073fa241a16ffd2553e0e9a5049c394fef00ddf5c2b8aa7091856e00a04ca0ffb6d500a47deec7439c2208a6a8e932fb1bd3148ac8156584997595bdcbcb38d

data/bin/osu-cc-scraper

@@ -0,0 +1,12 @@
+#!/usr/bin/env ruby
+
+require "osu-cc-scraper"
+require "csv"
+
+OsuCcScraper::Department.all.each do |department|
+  department.courses.each do |course|
+    course.sections.each do |section|
+      $stdout.puts(section.to_a.to_csv)
+    end
+  end
+end

data/lib/osu-cc-scraper/course.rb

@@ -0,0 +1,37 @@
+require "open-uri"
+require "nokogiri"
+require "osu-cc-scraper/section"
+
+class OsuCcScraper::Course
+  attr_accessor :subject_code
+  attr_accessor :course_number
+
+  def initialize(args)
+    args.each do |k,v|
+      instance_variable_set("@#{k}", v) unless v.nil?
+    end
+  end
+
+  def sections
+    html = OsuCcScraper::Section::fetch(self.subject_code, self.course_number)
+    OsuCcScraper::Section::parse(html)
+  end
+
+  private
+
+  def self.fetch(subject_code)
+    open("#{OsuCcScraper::ENDPOINT}/CourseList.aspx?subjectcode=#{subject_code}&level=undergrad&campus=corvallis").read
+  end
+
+  def self.parse(html)
+    ng = Nokogiri::HTML(html)
+    ng.css("tr td strong a:last").map{ |course|
+      uri = course.attribute('href').to_s.gsub('CourseDetail.aspx?', '')
+      params = URI.decode_www_form(uri).to_h
+      OsuCcScraper::Course.new({
+        subject_code: params["subjectcode"],
+        course_number: params["coursenumber"]
+      })
+    }
+  end
+end

data/lib/osu-cc-scraper/department.rb

@@ -0,0 +1,40 @@
+require "open-uri"
+require "nokogiri"
+require "osu-cc-scraper/course"
+
+class OsuCcScraper::Department
+  attr_accessor :subject_code
+
+  def initialize(args)
+    args.each do |k,v|
+      instance_variable_set("@#{k}", v) unless v.nil?
+    end
+  end
+
+  def self.all
+    html = OsuCcScraper::Department::fetch
+    OsuCcScraper::Department::parse(html)
+  end
+
+  def courses
+    html = OsuCcScraper::Course::fetch(self.subject_code)
+    OsuCcScraper::Course::parse(html)
+  end
+
+  private
+
+  def self.fetch
+    open("#{OsuCcScraper::ENDPOINT}/CourseDescription.aspx?level=undergrad").read
+  end
+
+  def self.parse(html)
+    ng = Nokogiri::HTML(html)
+    ng.xpath("//tr/td/font/a").map{ |department|
+      OsuCcScraper::Department.new({
+        subject_code: department.content[/\(.*?\)/][1..-2]
+      })
+    }.sort { |a,b|
+      a.subject_code.downcase <=> b.subject_code.downcase
+    }
+  end
+end

data/lib/osu-cc-scraper/section.rb

@@ -0,0 +1,64 @@
+require "open-uri"
+require "nokogiri"
+
+class OsuCcScraper::Section
+  attr_accessor :department
+  attr_accessor :number
+  attr_accessor :name
+  attr_accessor :term
+  attr_accessor :section
+  attr_accessor :instructor
+  attr_accessor :type
+  attr_accessor :status
+  attr_accessor :capacity
+  attr_accessor :availability
+
+  def initialize(args)
+    args.each do |k,v|
+      instance_variable_set("@#{k}", v) unless v.nil?
+    end
+  end
+
+  def to_a
+    [@department, @number, @name, @term, @section, @instructor, @type, @status, @capacity, @availability]
+  end
+
+  private
+
+  def self.fetch(subject_code, course_number)
+    open("#{OsuCcScraper::ENDPOINT}/CourseDetail.aspx?subjectcode=#{subject_code}&coursenumber=#{course_number}").read
+  end
+
+  def self.parse(html)
+    sections = []
+
+    ng = Nokogiri::HTML(html)
+    rows = ng.xpath('//table[@id="ctl00_ContentPlaceHolder1_SOCListUC1_gvOfferings"]/tr[position() > 1]')
+    rows.each_with_index do |row, key|
+      title = ng.at_xpath('//form/h3').content.split("\n")
+      query = "//table[@id='ctl00_ContentPlaceHolder1_SOCListUC1_gvOfferings']"
+
+      sections << OsuCcScraper::Section.new({
+        :department   => title[2].delete("\r\n").strip[0..-2].split(" ")[0],
+        :number       => title[2].split(" ")[1][0..-2],
+        :name         => title[3].strip,
+        :term         => at_xpath_or_null(ng, "#{query}/tr[position() = #{key+2}]/td[position() = 1]"),
+        :section      => at_xpath_or_null(ng, "#{query}/tr[position() = #{key+2}]/td[position() = 3]"),
+        :instructor   => at_xpath_or_null(ng, "#{query}/tr[position() = #{key+2}]/td[position() = 6]"),
+        :type         => at_xpath_or_null(ng, "#{query}/tr[position() = #{key+2}]/td[position() = 11]"),
+        :status       => at_xpath_or_null(ng, "#{query}/tr[position() = #{key+2}]/td[position() = 12]"),
+        :capacity     => at_xpath_or_null(ng, "#{query}/tr[position() = #{key+2}]/td[position() = 13]"),
+        :availability => at_xpath_or_null(ng, "#{query}/tr[position() = #{key+2}]/td[position() = 14]")
+      })
+    end
+    sections
+  end
+
+  def self.at_xpath_or_null(document, selector)
+    if document.at_xpath(selector) != nil
+      return document.at_xpath(selector).content.delete("\r\n").strip
+    else
+      return nil
+    end
+  end
+end

data/lib/osu-cc-scraper.rb

@@ -0,0 +1,7 @@
+class OsuCcScraper
+  ENDPOINT = "http://catalog.oregonstate.edu"
+end
+
+require "osu-cc-scraper/department"
+require "osu-cc-scraper/course"
+require "osu-cc-scraper/section"

metadata

@@ -0,0 +1,63 @@
+--- !ruby/object:Gem::Specification
+name: osu-cc-scraper
+version: !ruby/object:Gem::Version
+  version: 1.0.1
+platform: ruby
+authors:
+- Jonah George
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2015-08-10 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: nokogiri
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 1.6.6.2
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 1.6.6.2
+description: A script to gather course data from Oregon State's Course Catalog.
+email: jonah.george@me.com
+executables:
+- osu-cc-scraper
+extensions: []
+extra_rdoc_files: []
+files:
+- bin/osu-cc-scraper
+- lib/osu-cc-scraper.rb
+- lib/osu-cc-scraper/course.rb
+- lib/osu-cc-scraper/department.rb
+- lib/osu-cc-scraper/section.rb
+homepage: https://github.com/jonahgeorge/osu-cc-scraper
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.4.8
+signing_key: 
+specification_version: 4
+summary: OSU Course Catalog Scraper
+test_files: []