cricos_scrape 2.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 2cb41efd407804298343699766a9d530e4d51caf
+  data.tar.gz: 8ec6ecfb82d9ce2c2171297b8ef3026008e0eaaa
+SHA512:
+  metadata.gz: 47b66ae22e51f5846aa2aaac0bb4e93f4ceb1ce94d4ae81dab0179b0eaa95ed399d64bdf551c6a1416fac8059119fbd78292f9e963a9caf7364b5dcf5a090844
+  data.tar.gz: e2c8f522a4444ae73feff3d6ccc40c8c55017c1aa64ceec48cf12176c56e7876597f679cc6bcdfe8ba9a2a29182ad8746d7b2254b90321f09a975f3af4d6eecf

data/CONTRIBUTING.md

@@ -0,0 +1,51 @@
+Lotus is an open source project and we would love you to help us make it better.
+
+## Reporting Issues
+
+A well formatted issue is appreciated, and goes a long way in helping us help you.
+
+* Make sure you have a [GitHub account](https://github.com/signup/free)
+* Submit a [Github issue](./issues) by:
+  * Clearly describing the issue
+    * Provide a descriptive summary
+    * Explain the expected behavior
+    * Explain the actual behavior
+    * Provide steps to reproduce the actual behavior
+    * Provide your application's complete `Gemfile.lock` as text (in a [Gist](https://gist.github.com) for bonus points)
+    * Any relevant stack traces
+
+If you provide code, make sure it is formatted with the triple backticks (\`).
+
+At this point, we'd love to tell you how long it will take for us to respond,
+but we just don't know.
+
+## Pull requests
+
+We accept pull requests to Lotus for:
+
+* Adding documentation
+* Fixing bugs
+* Adding new features
+
+Not all features proposed will be added but we are open to having a conversation
+about a feature you are championing.
+
+Here's a quick guide:
+
+1. Fork the repo.
+
+2. Run the tests. This is to make sure your starting point works. Tests can be
+run via `rake`
+
+3. Create a new branch and make your changes. This includes tests for features!
+
+4. Push to your fork and submit a pull request. For more information, see
+[Github's pull request help section](https://help.github.com/articles/using-pull-requests/).
+
+At this point you're waiting on us. Expect a conversation regarding your pull
+request; Questions, clarifications, and so on.
+
+Some things that will increase the chance that your pull request is accepted:
+
+* Include tests that fail without your code, and pass with it
+* Update the documentation

data/Gemfile

@@ -0,0 +1,2 @@
+source 'https://rubygems.org'
+gemspec

data/Gemfile.lock

@@ -0,0 +1,64 @@
+PATH
+  remote: .
+  specs:
+    cricos_scrape (2.0)
+      mechanize (~> 2.7, >= 2.7.2)
+      slop (~> 4.2.0, >= 4.2.0)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    diff-lcs (1.2.5)
+    domain_name (0.5.25)
+      unf (>= 0.0.5, < 1.0.0)
+    http-cookie (1.0.2)
+      domain_name (~> 0.5)
+    mechanize (2.7.3)
+      domain_name (~> 0.5, >= 0.5.1)
+      http-cookie (~> 1.0)
+      mime-types (~> 2.0)
+      net-http-digest_auth (~> 1.1, >= 1.1.1)
+      net-http-persistent (~> 2.5, >= 2.5.2)
+      nokogiri (~> 1.4)
+      ntlm-http (~> 0.1, >= 0.1.1)
+      webrobots (>= 0.0.9, < 0.2)
+    mime-types (2.6.2)
+    mini_portile (0.6.2)
+    net-http-digest_auth (1.4)
+    net-http-persistent (2.9.4)
+    nokogiri (1.6.6.2)
+      mini_portile (~> 0.6.0)
+    ntlm-http (0.1.1)
+    rspec (3.3.0)
+      rspec-core (~> 3.3.0)
+      rspec-expectations (~> 3.3.0)
+      rspec-mocks (~> 3.3.0)
+    rspec-core (3.3.2)
+      rspec-support (~> 3.3.0)
+    rspec-expectations (3.3.1)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.3.0)
+    rspec-its (1.2.0)
+      rspec-core (>= 3.0.0)
+      rspec-expectations (>= 3.0.0)
+    rspec-mocks (3.3.2)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.3.0)
+    rspec-support (3.3.0)
+    slop (4.2.0)
+    unf (0.1.4)
+      unf_ext
+    unf_ext (0.0.7.1)
+    webrobots (0.1.1)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  bundler (~> 1.6)
+  cricos_scrape!
+  rspec (~> 3.3.0, >= 3.3.0)
+  rspec-its (~> 1.2.0, >= 1.2.0)
+
+BUNDLED WITH
+   1.10.6

data/LICENSE.md

@@ -0,0 +1,22 @@
+Copyright © 2014-2015 Trung Lê
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/Procfile

@@ -0,0 +1,3 @@
+job_1: bundle exec rake import:institutions MIN_ID=1 MAX_ID=10000
+job_2: bundle exec rake import:courses
+job_3: bundle exec rake import:contacts OVERWRITE=true

data/README.md

@@ -0,0 +1,40 @@
+[![Build Status](https://travis-ci.org/ruby-journal/cricos_scrape.rb.svg)](https://travis-ci.org/ruby-journal/cricos_scrape.rb)
+
+
+# CRICOS Scraper
+
+![CRICOS Logo](http://cricos.education.gov.au/images/cricos.gif)
+
+CRICOS lacks API for data retrieval (so are many government-based services). This gem
+helps scrape data from [http://cricos.education.gov.au](http://cricos.education.gov.au).
+
+This gem supports Ruby 2.2.3+ only.
+
+# Features
+
+Support scrapping following entities:
+
+* Institution
+* Course
+* Contact
+
+# Installation
+
+```
+gem install cricos_scrape
+```
+
+# Usage
+
+Please consult `cricos_scrape -h` command line.
+
+# Testing
+
+The tests are in the spec directory. Here syntax to test this gem.
+```
+rspec
+```
+
+# License
+
+MIT

data/Rakefile

@@ -0,0 +1,13 @@
+require 'rspec/core/rake_task'
+RSpec::Core::RakeTask.new
+
+require_relative 'lib/cricos_scrape'
+namespace :import do
+
+
+  task :contacts do
+    output_file  = ENV['OUTPUT_FILE'] || 'contacts.json'
+    CricosScrape::BulkImportContacts::new(output_file, ENV['OVERWRITE']).perform
+  end
+
+end

data/cricos_scrape.gemspec

@@ -0,0 +1,31 @@
+require './lib/cricos_scrape/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = 'cricos_scrape'
+  spec.version       = CricosScrape::VERSION
+  spec.authors       = ['Trung Lê', 'Toàn Lê']
+  spec.email         = ['trung.le@ruby-journal.com', 'ktoanlba@gmail.com']
+  spec.summary       = %q{CRICOS Scrape}
+  spec.description   = %q{Scrape Institutions, Courses, Contacts from CRICOS}
+  spec.homepage      = 'https://github.com/ruby-journal/cricos_scrape.rb'
+  spec.license       = 'MIT'
+
+  spec.files         = Dir['[A-Z]*',
+                        'lib/*.rb',
+                        'lib/cricos_scrape/*.rb',
+                        'spec/*.rb',
+                        'spec/fixtures/*.html']
+
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(spec)/})
+  spec.required_ruby_version = '>= 2.2.2'
+
+  spec.require_paths = ['lib']
+
+  spec.add_development_dependency 'bundler', '~> 1.6'
+  spec.add_development_dependency 'rspec', '~> 3.3.0', '>= 3.3.0'
+  spec.add_development_dependency 'rspec-its', '~> 1.2.0', '>= 1.2.0'
+
+  spec.add_runtime_dependency 'mechanize', '~> 2.7', '>= 2.7.2'
+  spec.add_runtime_dependency 'slop', '~> 4.2.0', '>= 4.2.0'
+end

data/lib/cricos_scrape/agent.rb

@@ -0,0 +1,9 @@
+require 'mechanize'
+
+module CricosScrape
+  def self.agent
+    agent = Mechanize.new
+    agent.user_agent = Mechanize::AGENT_ALIASES['Windows IE 6']
+    agent
+  end
+end

data/lib/cricos_scrape/bulk_import_courses.rb

@@ -0,0 +1,31 @@
+require_relative './importer/course_importer'
+require_relative './agent'
+
+module CricosScrape
+  class BulkImportCourses
+    def initialize(min_id=0, max_id=10000)
+      @range = (min_id..max_id).to_a
+      @agent = CricosScrape.agent
+    end
+
+    def perform
+      @range.each do |course_id|
+        scrape(course_id)
+      end
+    end
+
+    private
+
+    attr_reader :min_id, :max_id, :input, :agent
+
+    def scrape(course_id)
+      course = CourseImporter.new(agent, course_id: course_id).run
+
+      if course
+        puts course.to_json
+      else
+        STDERR.puts "Could not find course with Course ID #{course_id}"
+      end
+    end
+  end
+end

data/lib/cricos_scrape/bulk_import_institutions.rb

@@ -0,0 +1,31 @@
+require_relative './importer/institution_importer'
+require_relative './agent'
+
+module CricosScrape
+  class BulkImportInstitutions
+    def initialize(min_id=0, max_id=10000)
+      @range = (min_id..max_id).to_a
+      @agent = CricosScrape.agent
+    end
+
+    def perform
+      @range.each do |provider_id|
+        scrape(provider_id)
+      end
+    end
+
+    private
+
+    attr_reader :min_id, :max_id, :agent
+
+    def scrape(provider_id)
+      institution = InstitutionImporter.new(agent, provider_id: provider_id).run
+
+      if institution
+        puts institution.to_json
+      else
+        STDERR.puts "Could not find institution with Provider ID #{provider_id}"
+      end
+    end
+  end
+end

data/lib/cricos_scrape/import_contacts.rb

@@ -0,0 +1,22 @@
+require_relative './importer/contact_importer'
+require_relative './agent'
+
+module CricosScrape
+  class ImportContacts
+    def initialize
+      @agent = CricosScrape.agent
+    end
+
+    def perform
+      contacts = ContactImporter.new(@agent).run
+
+      if contacts.any?
+        contacts.each do |contact|
+          puts contact.to_json
+        end
+      else
+        STDERR.puts "Something not right, there is no Contacts returned"
+      end
+    end
+  end
+end

data/lib/cricos_scrape/json_struct.rb

@@ -0,0 +1,11 @@
+class Struct
+  def to_map
+    map = Hash.new
+    self.members.each { |m| map[m] = self[m] }
+    map
+  end
+
+  def to_json(*a)
+    to_map.to_json(*a)
+  end
+end

data/lib/cricos_scrape/version.rb

@@ -0,0 +1,3 @@
+module CricosScrape
+  VERSION = 2.0
+end

data/lib/cricos_scrape.rb

@@ -0,0 +1,8 @@
+require 'rubygems'
+require 'json'
+require 'json/add/core'
+
+require_relative 'cricos_scrape/json_struct'
+require_relative 'cricos_scrape/bulk_import_institutions'
+require_relative 'cricos_scrape/bulk_import_courses'
+require_relative 'cricos_scrape/import_contacts'

data/spec/contact_importer_spec.rb

@@ -0,0 +1,76 @@
+require 'spec_helper'
+
+describe CricosScrape::ContactImporter do
+
+  describe '#run' do
+    let(:agent) { CricosScrape.agent }
+    let(:importer) { CricosScrape::ContactImporter.new(agent) }
+    before do
+      stub_const('CricosScrape::ContactImporter::STATES_CODE', ['ACT', 'WA'])
+
+      allow(importer).to receive(:url_for).with('ACT').and_return(contact_details_of_state_act_uri)
+      allow(importer).to receive(:url_for).with('WA').and_return(contact_details_of_state_wa_uri)
+
+      @contacts = importer.run
+    end
+
+    context 'when the response body contains with states ACT and WA' do
+      it 'returns array contacts array' do
+        data = [
+          #contacts of ACT
+          CricosScrape::Contact.new('School Courses (and ELICOS and Foundation Programs where delivered by a school)',
+            'Ms Rebecca Hughes',
+            'ACT Education and Training Directorate',
+            CricosScrape::Address.new('GPO Box 158', nil, 'CANBERRA', 'ACT', '2601'),
+            '0262059299',
+            '',
+            'etd.contactus@act.gov.au'
+          ),
+          CricosScrape::Contact.new('Vocational Courses (and ELICOS courses offered by an RTO or remaining ‘stand-alone’ ELICOS provider)',
+            'ASQA Info Line',
+            'Australian Skills Quality Authority',
+            CricosScrape::Address.new('PO Box 9928', nil, 'Melbourne', 'VIC', '3001'),
+            '1300701801',
+            '',
+            'enquiries@asqa.gov.au'
+          ),
+          CricosScrape::Contact.new('Higher Education Courses (and ELICOS and  Foundation Programs where delivered in a pathway arrangement with a Higher Education Provider)',
+            'Tertiary Education Quality and Standards Agency',
+            'Tertiary Education Quality and Standards Agency',
+            CricosScrape::Address.new('GPO Box 1672', nil, 'Melbourne', 'VIC', '3001'),
+            '1300739585',
+            '1300739586',
+            'enquiries@teqsa.gov.au'
+          ),
+          #contacts of WA
+          CricosScrape::Contact.new('Vocational Courses (and ELICOS courses offered by an RTO or remaining ‘stand-alone’ ELICOS provider)',
+            'ASQA Info Line',
+            'Australian Skills Quality Authority',
+            CricosScrape::Address.new('PO Box 9928', nil, 'Melbourne', 'VIC', '3001'),
+            '1300701801',
+            '',
+            'enquiries@asqa.gov.au'
+          ),
+          CricosScrape::Contact.new('School Courses (and ELICOS and Foundation Programs where delivered by a school)',
+            'Mr Steve Page Senior Registration and Policy Officer',
+            'Department of Education Services, Non-Government & International Education Directorate',
+            CricosScrape::Address.new('PO Box 1766', nil, 'OSBORNE PARK', 'WA', '6916'),
+            '0894411962',
+            '0894411901',
+            'ngs@des.wa.gov.au'
+          ),
+          CricosScrape::Contact.new('Higher Education Courses (and ELICOS and  Foundation Programs where delivered in a pathway arrangement with a Higher Education Provider)',
+            'Tertiary Education Quality and Standards Agency',
+            'Tertiary Education Quality and Standards Agency',
+            CricosScrape::Address.new('GPO Box 1672', nil, 'Melbourne', 'VIC', '3001'),
+            '1300739585',
+            '1300739586',
+            'enquiries@teqsa.gov.au'
+          ),
+        ]
+
+        expect(@contacts).to eq data
+      end
+    end
+  end
+end

data/spec/course_importer_spec.rb

@@ -0,0 +1,71 @@
+require 'spec_helper'
+
+describe CricosScrape::CourseImporter do
+
+  describe '#run' do
+    let(:agent) { CricosScrape.agent }
+    subject(:course) { CricosScrape::CourseImporter.new(agent, course_id: 1).run }
+
+    before do
+      allow_any_instance_of(CricosScrape::CourseImporter).to receive(:url).and_return(uri)
+      course_list_page_1 = agent.get("#{uri}?LocationID=123")
+      course_list_page_2 = agent.get("#{uri}?LocationID=456")
+      allow_any_instance_of(Mechanize::Form).to receive(:submit).with(nil, {'action' => 'get-location-id'}).and_return(course_list_page_1, course_list_page_2)
+    end
+
+    context 'when there is no course found' do
+      let(:uri) { not_found_course_details_uri }
+
+      it 'does not import' do
+        expect(course).to be_nil
+      end
+    end
+
+    context 'when the details course is found' do
+      let(:uri) { course_details_without_pagination_uri }
+
+      its(:course_id) { is_expected.to eq 1 }
+      its(:course_name) { is_expected.to eq 'Primary Yrs K-6' }
+      its(:course_code) { is_expected.to eq '012395K' }
+      its(:dual_qualification) { is_expected.to eq 'No' }
+      its(:field_of_education) { is_expected.to eq '' }
+      its(:broad_field) { is_expected.to eq '12 - Mixed Field Programmes' }
+      its(:narrow_field) { is_expected.to eq '1201 - General Education Programmes' }
+      its(:detailed_field) { is_expected.to eq '120101 - General Primary and Secondary Education Programmes' }
+      its(:course_level) { is_expected.to eq 'Primary School Studies' }
+      its(:foundation_studies) { is_expected.to eq 'No' }
+      its(:work_component) { is_expected.to eq 'No' }
+      its(:course_language) { is_expected.to eq 'English' }
+      its(:duration) { is_expected.to eq '364' }
+      its(:total_cost) { is_expected.to eq '66,500' }
+      its(:contact_officers) do
+        contact_officers = [
+          CricosScrape::ContactOfficer.new('Principal Executive Officer', 'Nicole King', 'Manager', '0262056998', '62059239', nil),
+          CricosScrape::ContactOfficer.new('International Student Contact', 'PAUL Wang', 'Study Tour Coordinator', '62077293', '', 'paul.wang@act.gov.au'),
+        ]
+
+        is_expected.to eq contact_officers
+      end
+    end
+
+    context 'when the response body not contains pagination location' do
+      let(:uri) { course_details_without_pagination_uri }
+
+      its(:location_ids) do
+        location_ids = ["123", "456"]
+        is_expected.to eq location_ids
+      end
+    end
+
+    context 'when the contact officers contains table grid' do
+      let(:uri) { course_details_with_contact_officers_table_grid }
+      let(:data) { [CricosScrape::ContactOfficer.new('Principal Executive Officer', 'Andrew Vann', 'Vice-Chancellor', '02 6338 4209', '02 6338 4809', nil),
+        CricosScrape::ContactOfficer.new('International Student Contact', 'Matthew Evans', nil, '02 63657537', '02 63657590', 'mevans@csu.edu.au'),
+        CricosScrape::ContactOfficer.new('International Student Contact', 'Matthew Evans', nil, '02 6365 7537', '02 6365 7590', 'mevans@csu.edu.au')] }
+
+      its(:contact_officers) do
+        is_expected.to eq data
+      end
+    end
+  end
+end