regis-lector 0.9.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: cb2b34ab71515a7a6979b311e53851eb74227c83
+  data.tar.gz: e4b97553964c95e606698388ad0d5b61b48ad60b
+SHA512:
+  metadata.gz: 97569f6da326d5c86540cd2874eaf307a48c56a91c009da083b15dedd19f0731455c939f462ddee59f1da9c763fe910bd3ac4b464ab2d3041850160fc639c4a4
+  data.tar.gz: c6f69190688f7c02fdf5f055f181f57f5f713e5b52515e1fdb48ed326b23177689f493b69bf82655706ea7a6e71207c5544da6343dd02a518c9c14a1ca9566d5

data/.gitignore

@@ -0,0 +1,12 @@
+/.bundle/
+/.yardoc
+/Gemfile.lock
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/
+
+/data/config.yaml
+/data/veracross.json

data/.travis.yml

@@ -0,0 +1,5 @@
+sudo: false
+language: ruby
+rvm:
+  - 2.3.0
+before_install: gem install bundler -v 1.13.1

data/CODE_OF_CONDUCT.md

@@ -0,0 +1,74 @@
+# Contributor Covenant Code of Conduct
+
+## Our Pledge
+
+In the interest of fostering an open and welcoming environment, we as
+contributors and maintainers pledge to making participation in our project and
+our community a harassment-free experience for everyone, regardless of age, body
+size, disability, ethnicity, gender identity and expression, level of experience,
+nationality, personal appearance, race, religion, or sexual identity and
+orientation.
+
+## Our Standards
+
+Examples of behavior that contributes to creating a positive environment
+include:
+
+* Using welcoming and inclusive language
+* Being respectful of differing viewpoints and experiences
+* Gracefully accepting constructive criticism
+* Focusing on what is best for the community
+* Showing empathy towards other community members
+
+Examples of unacceptable behavior by participants include:
+
+* The use of sexualized language or imagery and unwelcome sexual attention or
+advances
+* Trolling, insulting/derogatory comments, and personal or political attacks
+* Public or private harassment
+* Publishing others' private information, such as a physical or electronic
+  address, without explicit permission
+* Other conduct which could reasonably be considered inappropriate in a
+  professional setting
+
+## Our Responsibilities
+
+Project maintainers are responsible for clarifying the standards of acceptable
+behavior and are expected to take appropriate and fair corrective action in
+response to any instances of unacceptable behavior.
+
+Project maintainers have the right and responsibility to remove, edit, or
+reject comments, commits, code, wiki edits, issues, and other contributions
+that are not aligned to this Code of Conduct, or to ban temporarily or
+permanently any contributor for other behaviors that they deem inappropriate,
+threatening, offensive, or harmful.
+
+## Scope
+
+This Code of Conduct applies both within project spaces and in public spaces
+when an individual is representing the project or its community. Examples of
+representing a project or community include using an official project e-mail
+address, posting via an official social media account, or acting as an appointed
+representative at an online or offline event. Representation of a project may be
+further defined and clarified by project maintainers.
+
+## Enforcement
+
+Instances of abusive, harassing, or otherwise unacceptable behavior may be
+reported by contacting the project team at thefrankmatranga@gmail.com. All
+complaints will be reviewed and investigated and will result in a response that
+is deemed necessary and appropriate to the circumstances. The project team is
+obligated to maintain confidentiality with regard to the reporter of an incident.
+Further details of specific enforcement policies may be posted separately.
+
+Project maintainers who do not follow or enforce the Code of Conduct in good
+faith may face temporary or permanent repercussions as determined by other
+members of the project's leadership.
+
+## Attribution
+
+This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4,
+available at [http://contributor-covenant.org/version/1/4][version]
+
+[homepage]: http://contributor-covenant.org
+[version]: http://contributor-covenant.org/version/1/4/

data/Gemfile

@@ -0,0 +1,8 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in lector.gemspec
+gemspec
+
+gem 'mysql2'
+gem 'activerecord'
+gem 'mechanize'

data/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2017 Frank Matranga
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2017 Frank Matranga
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,37 @@
+# Lector
+
+**Lector** is a Ruby gem written specifically for my high school's Moodle site. It can login as a student and then web scrape info on staff, teachers, students, and courses.
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'lector'
+```
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install lector
+
+## Usage
+
+Documentation can be found at TODO: LINK 
+
+## Development
+
+I still don't know much about real development of Ruby gems so if you want to contribute just clone, edit, and submit a pull request.
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at https://github.com/[USERNAME]/lector. This project is intended to be a safe, welcoming space for collaboration, and contributors are expected to adhere to the [Contributor Covenant](http://contributor-covenant.org) code of conduct.
+
+
+## License
+
+The gem is available as open source under the terms of the [MIT License](http://opensource.org/licenses/MIT).
+

data/Rakefile

@@ -0,0 +1,10 @@
+require "bundler/gem_tasks"
+require "rake/testtask"
+
+Rake::TestTask.new(:test) do |t|
+  t.libs << "test"
+  t.libs << "lib"
+  t.test_files = FileList['test/**/*_test.rb']
+end
+
+task :default => :test

data/bin/console

@@ -0,0 +1,19 @@
+#!/usr/bin/env ruby
+
+require "bundler/setup"
+require "lector"
+require "yaml"
+require "pp"
+
+PATH = "/home/frank/Projects/lector/data/config.yaml"
+CONFIG = YAML.load_file(PATH)
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+# require "pry"
+# Pry.start
+
+require "irb"
+IRB.start

data/bin/loop

@@ -0,0 +1,23 @@
+#!/usr/bin/env ruby
+
+require "bundler/setup"
+require "lector"
+require "yaml"
+require "pp"
+
+PATH = "/home/frank/Projects/lector/data/config.yaml"
+CONFIG = YAML.load_file(PATH)
+
+LECTOR = Lector.new(CONFIG)
+
+# LOOP THROUGH COURSES
+puts "LOOPING THROUGH COURSES"
+800.times do |id|
+  LECTOR.scrape_and_save(id + 1, :course)
+end
+
+# LOOP THROUGH PEOPLE
+puts "LOOPING THROUGH PEOPLE"
+3000.times do |id|
+  LECTOR.scrape_and_save(id + 1, :person)
+end

data/bin/setup

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+set -vx
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/lector.gemspec

@@ -0,0 +1,26 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+
+Gem::Specification.new do |spec|
+  spec.name          = "regis-lector"
+  spec.version       = "0.9.0"
+  spec.authors       = ["Frank Matranga"]
+  spec.email         = ["thefrankmatranga@gmail.com"]
+
+  spec.summary       = %q{Simple Moodle Scraper}
+  spec.description   = %q{Super simple Moodle site scraper for Frank Matranga's high school}
+  spec.homepage      = "https://github.com/Apexal/lector"
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files -z`.split("\x0").reject do |f|
+    f.match(%r{^(test|spec|features)/})
+  end
+  spec.bindir        = "exe"
+  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.require_paths = ["lib"]
+
+  spec.add_development_dependency "bundler", "~> 1.13"
+  spec.add_development_dependency "rake", "~> 10.0"
+  spec.add_development_dependency "minitest", "~> 5.0"
+end

data/lib/lector.rb

@@ -0,0 +1,63 @@
+require 'rubygems'
+
+require 'date'
+require 'yaml'
+require 'mysql2'
+require 'active_record'
+require 'mechanize'
+
+require 'bundler/setup'
+Bundler.setup(:default)
+
+# Require all modules
+Dir["#{File.dirname(__FILE__)}/lector/*.rb"].each { |file| require file }
+
+class Lector
+  @@defaults = {
+    db_host: 'localhost',
+    db_database: 'lector',
+    db_username: 'root',
+    db_password: '',
+    regis_username: '',
+    regis_password: '',
+    veracross_path: "#{Dir.pwd}/data/veracross.json"
+  }
+
+  VERSION = "0.9.0"
+
+  include Database
+  include Veracross
+  include Scraper
+
+  attr_reader :logged_in
+
+  def initialize(config)
+    puts "Lector v#{VERSION}"
+
+    # Fill in any blanks
+    @config = @@defaults.merge(config)
+    load_veracross
+
+    # First try to login to Moodle
+    login_to_moodle
+
+    # Connect to DB
+    connect_db
+
+    nil
+  end
+
+  def scrape_and_save(id, type)
+    puts "Scraping and saving #{type.to_s} with Moodle ID #{id}"
+    begin
+      return case type
+        when :person
+          save_person(extract_person(id))
+        when :course
+          save_course(extract_course(id))
+      end
+    rescue => e
+      puts "Failed to scrape and save #{type.to_s} with ID: #{id}\n#{e}"
+    end
+  end
+end

data/lib/lector/database.rb

@@ -0,0 +1,91 @@
+module Database
+  def connect_db
+    puts "Connecting to MySQL DB at #{@config[:db_host]}"
+    ActiveRecord::Base.establish_connection(
+      adapter: 'mysql2',
+      host: @config[:db_host],
+      database: @config[:db_database],
+      username: @config[:db_username],
+      password: @config[:db_password]
+    )
+  end
+
+  class Student < ActiveRecord::Base
+    has_and_belongs_to_many :courses
+  end
+
+  class Course < ActiveRecord::Base
+    belongs_to :staff, inverse_of: :courses, foreign_key: 'teacher_id'
+    has_and_belongs_to_many :students
+  end
+
+  class Staff < ActiveRecord::Base
+    has_and_belongs_to_many :courses
+  end
+
+  # Determine staff or student
+  def save_person(info)
+    return case info[:type]
+      when :student
+        save_student(info)
+      else
+        save_staff(info)
+      end
+  end
+
+  def save_course(info)
+    info.delete(:type)
+
+    # DB throws error if doesn't exist so resort to nil if throws
+    course = Course.find(info[:id]) rescue nil
+
+    if course.nil?  
+      course = Course.create(info)
+    else
+      course.update(info)
+    end
+
+    course.save!
+    course
+  end
+
+  def save_staff(info)
+    info.delete(:type)
+
+    # DB throws error if doesn't exist so resort to nil if throws
+    staff = Staff.find(info[:id]) rescue nil
+
+    course_ids = info.delete(:course_ids)
+
+    if staff.nil?
+      staff = Staff.create(info)
+    else
+      staff.update(info)
+    end
+
+    staff.courses = Course.where(id: course_ids)  # Throws error if courses don't exist
+
+    staff.save!
+    staff
+  end
+
+  def save_student(info)
+    info.delete(:type)
+
+    # DB throws error if doesn't exist so resort to nil if throws
+    student = Student.find(info[:id]) rescue nil
+
+    course_ids = info.delete(:course_ids)
+
+    if student.nil?
+      student = Student.create(info)
+    else
+      student.update(info)
+    end
+
+    student.courses = Course.where(id: course_ids) # Throws error if courses don't exist
+
+    student.save!
+    student
+  end
+end

data/lib/lector/exceptions.rb

@@ -0,0 +1,7 @@
+module Exceptions
+  class InvalidCredentialsError < StandardError  
+  end
+
+  class InvalidPageError < StandardError
+  end
+end

data/lib/lector/other/storedata.rb

@@ -0,0 +1,14 @@
+module StoreData
+  # Serializes an object and saves it to file.
+  def save_to_file(file, object)
+    File.open(file, 'w') do |f|
+      f.write YAML.dump(object)
+    end
+  end
+
+  # Loads yaml from file or returns an empty hash if file doesn't exist.
+  def load_file(file)
+    return YAML.load_file(file) if File.exist?(file)
+    {}
+  end
+end

data/lib/lector/scraper.rb

@@ -0,0 +1,125 @@
+module Scraper
+  include Exceptions
+  include Veracross
+
+  # Constants
+  AGENT = Mechanize.new
+  AGENT.user_agent_alias = 'Linux Firefox'
+  
+  GRADES = %w(Freshmen Sophomores Juniors Seniors).freeze
+  SINGULAR_GRADES = %w(Freshman Sophomore Junior Senior).freeze
+  ADVISEMENT_TO_GRAD_YEAR = {'1' => 20, '2' => 19, '3' => 18, '4' => 17}
+  COURSE_TITLE_IGNORES = %w(Department REACH Advisement Office NYSAIS)
+  SUCCESSFUL_LOGIN_PAGE_TITLE = 'Dashboard'
+  IGNORE_TITLES = %w(Notice Error Test Parent Nurse Student)
+
+  def login_to_moodle
+    puts "Attempting to login to Moodle as #{@config[:regis_username]}..."
+    puts AGENT
+    page = AGENT.post('https://moodle.regis.org/login/index.php', {
+      username: @config[:regis_username],
+      password: @config[:regis_password],
+    })
+
+    raise InvalidCredentialsError, 'Invalid Regis credentials! Couldn\'t log into Moodle.' unless page.title == SUCCESSFUL_LOGIN_PAGE_TITLE
+
+    true
+  end
+
+  def get_profile(id, type)
+    base_url = (type == :person ? 'http://moodle.regis.org/user/profile.php?id=' : 'http://moodle.regis.org/course/view.php?id=')
+
+    # Request page
+    page = AGENT.get(base_url + id.to_s)
+
+    # Discard unneccessary pages
+    raise InvalidPageError, "Invalid page: '#{page.title}'" if page.title.nil? || IGNORE_TITLES.any? { |w| page.title.downcase.include?(w.downcase) }
+    raise InvalidPageError, "Name is too short: '#{page.title}'" if page.title.split(':').length < 2
+
+    page
+  end
+
+  def extract_person(id)
+    page = get_profile(id, :person)
+    
+    title = page.title
+
+    name = title.split(':')[0].split(' ')
+    first_name = name[0]
+    last_name = name[1..-1].join(' ')
+
+    picture_url = nil
+    begin
+      picture_url = page.search("a/img[@alt=\"Picture of #{first_name} #{last_name}\"]")[0]['src']
+    rescue => e
+      puts e
+      raise InvalidPageError, 'Page doesn\'t have picture.'
+    end
+
+    # DEPARTMENT (advisement for students, subject for staff)
+    department = page.search("//dd[../dt = 'Department']/text()").to_s
+
+    type = (/\A\d+\z/.match(department[0]) || department.empty? ? :student : :staff)
+
+    username_guess = (type == :staff ? "#{first_name[0]}#{last_name}" : "#{first_name[0]}#{last_name}#{Lector::ADVISEMENT_TO_GRAD_YEAR[department[0]]}").downcase.gsub("'", '').gsub('-', '').gsub(' ', '')
+
+    returning = {
+      id: id,
+      type: type,
+      first_name: first_name,
+      last_name: last_name,
+      email: username_guess + '@regis.org',
+      department: department,
+      pictureurl: picture_url,
+      username: username_guess,
+      course_ids: page.search("//dd/ul/li/a[contains(@href, 'http://moodle.regis.org/course/view.php?id=')]").map { |link| link["href"].split("id=")[1].split("&")[0] }
+    }
+
+    if type == :student
+      # FIND INFO FROM VERACROSS
+      info = find_by_username_or_email(username_guess)
+      raise InvalidPageError, 'Student doesn\'t exist. Failed to link to Veracross data.' if info.nil?
+
+      returning[:advisement] = returning.delete(:department)
+      returning[:veracross_id] = info['person_pk']
+      returning[:pictureurl] = info['photo_url']
+      returning[:graduation_year] = info['graduation_year']
+      returning[:address] = info['resident_address'].sub('<br />', ' ')
+      returning[:birthday] = Date.parse(info['birthday'])
+    end
+
+    puts "#{id}: #{type.capitalize} #{last_name}, #{first_name} of #{department} (#{username_guess})"
+
+    returning
+  end
+
+  def extract_course(id)
+    page = get_profile(id, :course)
+
+    parts = page.title.split(':')
+    is_class = parts.length > 2 # Classes have a teacher, title would have ': Teacher Name'
+
+    # Course: Theater Production: Grunner
+    title = (parts.length == 1 ? parts[0]  : parts[1]).strip
+
+    raise "#{id}: Ignoring 'course' #{title}" if COURSE_TITLE_IGNORES.any? { |ignore| title.end_with?(ignore) || title.start_with?(ignore) }
+
+    returning = {
+      type: :course,
+      id: id,
+      title: title,
+      is_class: is_class
+    }
+
+    if is_class
+      # Find teacher
+      teacher_page = AGENT.get("http://moodle.regis.org/user/index.php?roleid=3&sifirst=&silast=&id=#{id.to_s}")
+      returning[:teacher_id] = teacher_page.search("//strong/a[contains(@href, 'moodle.regis.org')]")[0]['href'].split("?id=")[1].split("&course=")[0].to_i
+    end
+
+    puts "#{id}: Course #{parts[1..-1].join}#{" (class)" if is_class}"
+
+    returning
+  end
+
+end

data/lib/lector/veracross.rb

@@ -0,0 +1,25 @@
+module Veracross
+  # USEFUL PROPERTIES
+  # first_name
+  # last_name
+  # birthday
+  # grade_level
+  # graduation_year
+  # advisor
+  # address
+  # email
+
+  def load_veracross
+    puts "Loading Veracross data from #{@config[:veracross_path]}"
+    path = @config[:veracross_path]
+    @data = JSON.load(File.read(path))
+
+    puts "Loaded #{@data.length} students"
+  end
+
+  def find_by_username_or_email(u_or_e)
+    load if @data.nil?
+    email = (u_or_e.end_with?('@regis.org') ? u_or_e : u_or_e + '@regis.org')
+    @data.find { |entry| entry['email'] == email }
+  end
+end

data/run.rb

@@ -0,0 +1 @@
+require_relative 'lib/lector.rb'

metadata

@@ -0,0 +1,105 @@
+--- !ruby/object:Gem::Specification
+name: regis-lector
+version: !ruby/object:Gem::Version
+  version: 0.9.0
+platform: ruby
+authors:
+- Frank Matranga
+autorequire: 
+bindir: exe
+cert_chain: []
+date: 2017-06-08 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.13'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.13'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+- !ruby/object:Gem::Dependency
+  name: minitest
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.0'
+description: Super simple Moodle site scraper for Frank Matranga's high school
+email:
+- thefrankmatranga@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".travis.yml"
+- CODE_OF_CONDUCT.md
+- Gemfile
+- LICENSE
+- LICENSE.txt
+- README.md
+- Rakefile
+- bin/console
+- bin/loop
+- bin/setup
+- lector.gemspec
+- lib/lector.rb
+- lib/lector/database.rb
+- lib/lector/exceptions.rb
+- lib/lector/other/storedata.rb
+- lib/lector/scraper.rb
+- lib/lector/veracross.rb
+- run.rb
+homepage: https://github.com/Apexal/lector
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.6.12
+signing_key: 
+specification_version: 4
+summary: Simple Moodle Scraper
+test_files: []