osu-cc-scraper 1.0.4 → 2.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 00e547e50b759e05a38eb3251f333ef13acda4bb
-  data.tar.gz: 2fe51e85192c1e79df01eba6392d680c1979c8ae
+  metadata.gz: 71df7f91c3049b5d62323435cb066f324c2882ab
+  data.tar.gz: 02eb25347ade8940adabd831f87cbd631c943f1b
 SHA512:
-  metadata.gz: 6b07ba4686e5d1ff8fad54f92d4dbf13372fc4cba300d3c71fa76683fcb832703a04f7db8e1b0b0e39b263df67cc6ee780c49f303b4f6b14ab1d5bc40918948e
-  data.tar.gz: 2ac1f329ec36729cbc347fe3cd59cb639a5867dde8ed9e13e284e1b3e37d71b3c67cdf497687915241f46ced0cf185e18ad1344dcaac4917ddf2357b6ce9edbe
+  metadata.gz: 1a2017bbf88c24da39b941bbd69bacc3a8cf5fe1309e28a530649bbac78de660a17d60b6b068e91e7d037f05950d28cd33e7f530b9765acc1ee4081c683061fd
+  data.tar.gz: 9573bdcb5255459cdfda6072ec943d9001915ef6fc97860177fa0bd3358acd145bbb75335372d7f8267bad1bccbad60df247cc4fc32b399f41f4643d7657e127

data/lib/osu-cc-scraper.rb

@@ -2,6 +2,7 @@ module OsuCcScraper
   ENDPOINT = "http://catalog.oregonstate.edu"
 end
 
+require "osu-cc-scraper/university"
 require "osu-cc-scraper/department"
 require "osu-cc-scraper/course"
 require "osu-cc-scraper/section"

data/lib/osu-cc-scraper/course.rb

@@ -1,43 +1,90 @@
 require "open-uri"
-require "nokogiri"
+require "oga"
+
 require "osu-cc-scraper/section"
 
-class OsuCcScraper::Course
-  # @return [String]
-  attr_accessor :subject_code
-  # @return [String]
-  attr_accessor :course_number
-  # @return [String]
-  attr_accessor :name
-
-  # @return [OsuCcScraper::Course]
-  def initialize(args)
-    args.each do |k,v|
-      instance_variable_set("@#{k}", v) unless v.nil?
+module OsuCcScraper
+  class Course < Struct.new(:subject_code, :course_number, :name)
+
+    def sections
+      html = fetch_sections
+      parse_sections(html)
     end
-  end
 
-  # @return [Array<OsuCcScraper::Section>]
-  def sections
-    html = OsuCcScraper::Section::fetch(self.subject_code, self.course_number)
-    OsuCcScraper::Section::parse(html)
-  end
+    private
 
-  private
+      SECTIONS_XPATH = "//table[@id='ctl00_ContentPlaceHolder1_SOCListUC1_gvOfferings']//tr[position() > 1]"
+      TITLE_XPATH = "//form/h3"
 
-  def self.fetch(subject_code)
-    open("#{OsuCcScraper::ENDPOINT}/CourseList.aspx?subjectcode=#{subject_code}&level=undergrad&campus=corvallis").read
-  end
+      def fetch_sections
+        open("#{ENDPOINT}/CourseDetail.aspx?subjectcode=#{subject_code}&coursenumber=#{course_number}").read
+      end
+
+      def parse_sections(html)
+        document = Oga.parse_html(html)
+
+        document.xpath(SECTIONS_XPATH).map { |row| 
+          Section.new(
+            parse_department(document), 
+            parse_number(document), 
+            parse_name(document), 
+            parse_term(row), 
+            parse_section(row),
+            parse_instructor(row), 
+            parse_type(row), 
+            parse_status(row),
+            parse_capacity(row), 
+            parse_current(row)
+          )
+        }
+      end
+
+      def fetch_column(document, selector)
+        document.xpath(selector)&.text&.delete("\r\n")&.strip
+      end
+
+      def parse_department(document)
+        title = document.xpath(TITLE_XPATH).text.split("\n")
+        title[2].delete("\r\n").strip[0..-2].split(" ")[0]
+      end
+
+      def parse_number(document)
+        title = document.xpath(TITLE_XPATH).text.split("\n")
+        title[2].split(" ")[1][0..-2]
+      end
+
+      def parse_name(document)
+        title = document.xpath(TITLE_XPATH).text.split("\n")
+        title[3].strip
+      end
+
+      def parse_term(row)
+        fetch_column(row, "td[position() = 1]")
+      end
+
+      def parse_section(row)
+        fetch_column(row, "td[position() = 3]")
+      end
+
+      def parse_instructor(row)
+        fetch_column(row, "td[position() = 6]")
+      end
+
+      def parse_type(row)
+        fetch_column(row, "td[position() = 11]")
+      end
+
+      def parse_status(row)
+        fetch_column(row, "td[position() = 12]")
+      end
+
+      def parse_capacity(row)
+        fetch_column(row, "td[position() = 13]")&.to_i
+      end
+
+      def parse_current(row)
+        fetch_column(row, "td[position() = 14]")&.to_i
+      end
 
-  def self.parse(html)
-    ng = Nokogiri::HTML(html)
-    ng.css("tr td strong a:last").map{ |course|
-      # NE 311H INTRODUCTION TO THERMAL-FLUID SCIENCES (4)
-      parts = course.text.split(' ')
-      OsuCcScraper::Course.new \
-        subject_code:  parts[0],
-        course_number: parts[1],
-        name:          parts[2..parts.length-2].join(" ")
-    }
   end
 end

data/lib/osu-cc-scraper/department.rb

@@ -1,47 +1,45 @@
 require "open-uri"
-require "nokogiri"
+require "oga"
+
 require "osu-cc-scraper/course"
 
-class OsuCcScraper::Department
-  # @return [String]
-  attr_accessor :name
-  # @return [String]
-  attr_accessor :subject_code
+module OsuCcScraper
+  class Department < Struct.new(:name, :subject_code)
 
-  # @return [OsuCcScraper::Department]
-  def initialize(args)
-    args.each do |k,v|
-      instance_variable_set("@#{k}", v) unless v.nil?
+    def courses
+      html = fetch_courses
+      parse_courses(html)
     end
-  end
 
-  # @return [Array<OsuCcScraper::Department>]
-  def self.all
-    html = OsuCcScraper::Department::fetch
-    OsuCcScraper::Department::parse(html)
-  end
+    private
 
-  # @return [Array<OsuCcScraper::Course>]
-  def courses
-    html = OsuCcScraper::Course::fetch(self.subject_code)
-    OsuCcScraper::Course::parse(html)
-  end
+      def fetch_courses
+        open("#{ENDPOINT}/CourseList.aspx?subjectcode=#{subject_code}&level=undergrad&campus=corvallis").read
+      end
 
-  private
+      def parse_courses(html)
+        document = Oga.parse_html(html)
+        document.xpath("//tr//td//strong/a[last()]").map { |row|
+          Course.new(
+            parse_course_subject_code(row),
+            parse_course_course_number(row),
+            parse_course_name(row)
+          )
+        }
+      end
 
-  def self.fetch
-    open("#{OsuCcScraper::ENDPOINT}/CourseDescription.aspx?level=undergrad").read
-  end
+      # NE 311H INTRODUCTION TO THERMAL-FLUID SCIENCES (4)
+      def parse_course_subject_code(row)
+        row.text.split(' ')[0]
+      end
+
+      def parse_course_course_number(row)
+        row.text.split(' ')[1]
+      end
+
+      def parse_course_name(row)
+        row.text.split(' ')[2..-2].join(" ")
+      end
 
-  def self.parse(html)
-    ng = Nokogiri::HTML(html)
-    ng.xpath("//tr/td/font/a").map{ |department|
-      OsuCcScraper::Department.new({
-        subject_code: department.content[/\(.*?\)/][1..-2],
-        name:         department.content[/([^(]+)/].strip
-      })
-    }.sort { |a,b|
-      a.subject_code.downcase <=> b.subject_code.downcase
-    }
   end
 end

data/lib/osu-cc-scraper/section.rb

@@ -1,76 +1,5 @@
-require "open-uri"
-require "nokogiri"
-
-class OsuCcScraper::Section
-  # @return [String]
-  attr_accessor :department
-  # @return [String]
-  attr_accessor :number
-  # @return [String]
-  attr_accessor :name
-  # @return [String]
-  attr_accessor :term
-  # @return [String]
-  attr_accessor :section
-  # @return [String]
-  attr_accessor :instructor
-  # @return [String]
-  attr_accessor :type
-  # @return [String]
-  attr_accessor :status
-  # @return [String]
-  attr_accessor :capacity
-  # @return [String]
-  attr_accessor :availability
-
-  # @return [OsuCcScraper::Section]
-  def initialize(args)
-    args.each do |k,v|
-      instance_variable_set("@#{k}", v) unless v.nil?
-    end
-  end
-
-  # @return [Array]
-  def to_a
-    [@department, @number, @name, @term, @section, @instructor, @type, @status, @capacity, @availability]
-  end
-
-  private
-
-  def self.fetch(subject_code, course_number)
-    open("#{OsuCcScraper::ENDPOINT}/CourseDetail.aspx?subjectcode=#{subject_code}&coursenumber=#{course_number}").read
-  end
-
-  def self.parse(html)
-    sections = []
-
-    ng = Nokogiri::HTML(html)
-    rows = ng.xpath('//table[@id="ctl00_ContentPlaceHolder1_SOCListUC1_gvOfferings"]/tr[position() > 1]')
-    rows.each_with_index do |row, key|
-      title = ng.at_xpath('//form/h3').content.split("\n")
-      query = "//table[@id='ctl00_ContentPlaceHolder1_SOCListUC1_gvOfferings']"
-
-      sections << OsuCcScraper::Section.new({
-        :department   => title[2].delete("\r\n").strip[0..-2].split(" ")[0],
-        :number       => title[2].split(" ")[1][0..-2],
-        :name         => title[3].strip,
-        :term         => at_xpath_or_null(ng, "#{query}/tr[position() = #{key+2}]/td[position() = 1]"),
-        :section      => at_xpath_or_null(ng, "#{query}/tr[position() = #{key+2}]/td[position() = 3]"),
-        :instructor   => at_xpath_or_null(ng, "#{query}/tr[position() = #{key+2}]/td[position() = 6]"),
-        :type         => at_xpath_or_null(ng, "#{query}/tr[position() = #{key+2}]/td[position() = 11]"),
-        :status       => at_xpath_or_null(ng, "#{query}/tr[position() = #{key+2}]/td[position() = 12]"),
-        :capacity     => at_xpath_or_null(ng, "#{query}/tr[position() = #{key+2}]/td[position() = 13]"),
-        :availability => at_xpath_or_null(ng, "#{query}/tr[position() = #{key+2}]/td[position() = 14]")
-      })
-    end
-    sections
-  end
-
-  def self.at_xpath_or_null(document, selector)
-    if document.at_xpath(selector) != nil
-      return document.at_xpath(selector).content.delete("\r\n").strip
-    else
-      return nil
-    end
+module OsuCcScraper
+  class Section < Struct.new(:department, :number, :name, :term, :section, 
+                             :instructor, :type, :status, :capacity, :current)
   end
 end

data/lib/osu-cc-scraper/university.rb

@@ -0,0 +1,37 @@
+require "open-uri"
+require "oga"
+
+require "osu-cc-scraper/department"
+
+module OsuCcScraper
+  class University
+
+    def departments
+      html = fetch_departments
+      parse_departments(html)
+    end
+
+    def fetch_departments
+      open("#{ENDPOINT}/CourseDescription.aspx?level=undergrad").read
+    end
+
+    def parse_departments(html)
+      ng = Oga.parse_html(html)
+      ng.xpath("//tr/td/font/a").map { |row|
+        Department.new(
+          parse_department_name(row),
+          parse_department_subject_code(row), 
+        )
+      }
+    end
+
+    def parse_department_subject_code(row)
+      row.text[/\(.*?\)/][1..-2]    
+    end
+
+    def parse_department_name(row)
+      row.text[/([^(]+)/].strip    
+    end
+
+  end
+end

metadata

@@ -1,30 +1,59 @@
 --- !ruby/object:Gem::Specification
 name: osu-cc-scraper
 version: !ruby/object:Gem::Version
-  version: 1.0.4
+  version: 2.0.0
 platform: ruby
 authors:
 - Jonah George
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-12-18 00:00:00.000000000 Z
+date: 2016-02-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  name: nokogiri
+  name: oga
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '='
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.6.6.2
+        version: 2.1.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '='
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.6.6.2
-description: A script to gather course data from Oregon State's Course Catalog.
+        version: 2.1.0
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: A script to gather course data from Oregon State University's Course
+  Catalog.
 email: jonah.george@me.com
 executables: []
 extensions: []
@@ -34,6 +63,7 @@ files:
 - lib/osu-cc-scraper/course.rb
 - lib/osu-cc-scraper/department.rb
 - lib/osu-cc-scraper/section.rb
+- lib/osu-cc-scraper/university.rb
 homepage: https://github.com/jonahgeorge/osu-cc-scraper
 licenses:
 - MIT
@@ -46,7 +76,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: '0'
+      version: 2.3.0
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
@@ -54,7 +84,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.4.5.1
+rubygems_version: 2.5.1
 signing_key: 
 specification_version: 4
 summary: OSU Course Catalog Scraper