cricos_scrape 2.0 → 2.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 2cb41efd407804298343699766a9d530e4d51caf
-  data.tar.gz: 8ec6ecfb82d9ce2c2171297b8ef3026008e0eaaa
+  metadata.gz: 3961f2418e9bc173a4412c6d16a83408dfe2c280
+  data.tar.gz: fdf7c3c46869cd7e805a05e246b36649cacfaa19
 SHA512:
-  metadata.gz: 47b66ae22e51f5846aa2aaac0bb4e93f4ceb1ce94d4ae81dab0179b0eaa95ed399d64bdf551c6a1416fac8059119fbd78292f9e963a9caf7364b5dcf5a090844
-  data.tar.gz: e2c8f522a4444ae73feff3d6ccc40c8c55017c1aa64ceec48cf12176c56e7876597f679cc6bcdfe8ba9a2a29182ad8746d7b2254b90321f09a975f3af4d6eecf
+  metadata.gz: 698d1a4564472abee988d847e06c93f93f718102036b5eb3ddd68a795c86b6827302b21da9db2490b7a647922227d38da9551a76430ff593ffc9c7eed6a848e6
+  data.tar.gz: c4a81a6e2d005d3047f2170a85e66c4c10bdaf73196b068e5ddeb2a3fe0bfd14f1fe4a9a5e7d263aad08beb314c2532d41e4b16524aee47c3e5813a755a7f772

data/README.md

@@ -8,7 +8,7 @@
 CRICOS lacks API for data retrieval (so are many government-based services). This gem
 helps scrape data from [http://cricos.education.gov.au](http://cricos.education.gov.au).
 
-This gem supports Ruby 2.2.3+ only.
+This gem supports MRI Ruby 2.0.0+.
 
 # Features
 

data/bin/cricos_scrape

@@ -0,0 +1,40 @@
+#!/usr/bin/env ruby
+
+require 'cricos_scrape'
+require 'cricos_scrape/version'
+require 'commander/import'
+
+program :version, CricosScrape::VERSION.to_s
+program :description, 'Scrape data from CRICOS website'
+
+command :scrape do |c|
+  c.syntax = 'cricos_scrape scrape [institutions|courses|contacts] [min_id:1] [max_id:10000]'
+  c.summary = 'Scrape entities from CRICOS'
+  c.description = c.summary
+  c.example 'Import institutions', 'cricos_scrape scrape institutions'
+  c.example 'Import institutions and persist to a file', 'cricos_scrape scrape institutions >> institutions.json'
+  c.example 'Import institutions with specified ID range [1-200]', 'cricos_scrape scrape institutions 1 200'
+  c.example 'Import courses', 'cricos_scrape scrape courses'
+  c.example 'Import courses and persist to a file', 'cricos_scrape scrape courses >> courses.json'
+  c.example 'Import courses with specified ID range [1-200]', 'cricos_scrape scrape courses 1 200'
+  c.example 'Import contacts', 'cricos_scrape scrape contacts. NOTE: ID range option does not apply'
+  c.example 'Import contacts and persist to a file', 'cricos_scrape scrape contacts >> contacts.json'
+
+  c.action do |args, options|
+    entity = args[0]
+    min_id = args[1] || 1
+    max_id = args[2] || 10000
+
+    case entity
+    when 'institutions'
+      CricosScrape::BulkImportInstitutions.new(min_id, max_id).perform
+    when 'courses'
+      CricosScrape::BulkImportCourses.new(min_id, max_id).perform
+    when 'contacts'
+      CricosScrape::ImportContacts.new.perform
+    else
+      STDERR.puts "[ERROR] Invalid entity. Please see `cricos_scrape scrape -h` for more details"
+      exit 1
+    end
+  end
+end

data/cricos_scrape.gemspec

@@ -1,4 +1,7 @@
-require './lib/cricos_scrape/version'
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'cricos_scrape/version'
 
 Gem::Specification.new do |spec|
   spec.name          = 'cricos_scrape'
@@ -10,15 +13,10 @@ Gem::Specification.new do |spec|
   spec.homepage      = 'https://github.com/ruby-journal/cricos_scrape.rb'
   spec.license       = 'MIT'
 
-  spec.files         = Dir['[A-Z]*',
-                        'lib/*.rb',
-                        'lib/cricos_scrape/*.rb',
-                        'spec/*.rb',
-                        'spec/fixtures/*.html']
-
-  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.files         = `git ls-files -z -- lib/* bin/* LICENSE.md README.md cricos_scrape.gemspec`.split("\x0")
+  spec.executables   = ['cricos_scrape']
   spec.test_files    = spec.files.grep(%r{^(spec)/})
-  spec.required_ruby_version = '>= 2.2.2'
+  spec.required_ruby_version = '>= 2.0.0'
 
   spec.require_paths = ['lib']
 

data/lib/cricos_scrape/bulk_import_courses.rb

@@ -1,5 +1,5 @@
-require_relative './importer/course_importer'
-require_relative './agent'
+require 'cricos_scrape/importer/course_importer'
+require 'cricos_scrape/agent'
 
 module CricosScrape
   class BulkImportCourses

data/lib/cricos_scrape/bulk_import_institutions.rb

@@ -1,5 +1,5 @@
-require_relative './importer/institution_importer'
-require_relative './agent'
+require 'cricos_scrape/importer/institution_importer'
+require 'cricos_scrape/agent'
 
 module CricosScrape
   class BulkImportInstitutions

data/lib/cricos_scrape/entities/address.rb

@@ -0,0 +1,4 @@
+module CricosScrape
+  class Address < Struct.new(:address_line_1, :address_line_2, :suburb, :state, :postcode)
+  end
+end

data/lib/cricos_scrape/entities/contact.rb

@@ -0,0 +1,4 @@
+module CricosScrape
+  class Contact < Struct.new(:type_of_course, :name, :organisation, :postal_address, :telephone, :facsimile, :email)
+  end
+end

data/lib/cricos_scrape/entities/contact_officer.rb

@@ -0,0 +1,4 @@
+module CricosScrape
+  class ContactOfficer < Struct.new(:role, :name, :title, :phone, :fax, :email)
+  end
+end

data/lib/cricos_scrape/entities/course.rb

@@ -0,0 +1,4 @@
+module CricosScrape
+  class Course < Struct.new(:course_id, :course_name, :course_code, :dual_qualification, :field_of_education, :broad_field, :narrow_field, :detailed_field, :course_level, :foundation_studies, :work_component, :course_language, :duration, :total_cost, :contact_officers, :location_ids)
+  end
+end

data/lib/cricos_scrape/entities/institution.rb

@@ -0,0 +1,4 @@
+module CricosScrape
+  class Institution < Struct.new(:provider_id, :provider_code, :trading_name, :name, :type, :total_capacity, :postal_address, :website, :locations, :contact_officers)
+  end
+end

data/lib/cricos_scrape/entities/location.rb

@@ -0,0 +1,4 @@
+module CricosScrape
+  class Location < Struct.new(:location_id, :name, :state, :number_of_courses)
+  end
+end

data/lib/cricos_scrape/import_contacts.rb

@@ -1,5 +1,5 @@
-require_relative './importer/contact_importer'
-require_relative './agent'
+require 'cricos_scrape/importer/contact_importer'
+require 'cricos_scrape/agent'
 
 module CricosScrape
   class ImportContacts

data/lib/cricos_scrape/importer/contact_importer.rb

@@ -0,0 +1,120 @@
+require 'cricos_scrape/entities/contact'
+require 'cricos_scrape/entities/address'
+
+module CricosScrape
+  class ContactImporter
+
+    CONTACT_URL = 'http://cricos.education.gov.au/Contacts/CRICOSContacts.aspx'
+    STATES_CODE = ['ACT', 'NSW', 'NT', 'QLD', 'SA', 'TAS', 'VIC', 'WA']
+
+    def initialize(agent)
+      @agent = agent
+    end
+
+    def run
+      contacts = []
+
+      for state in STATES_CODE
+        @page = agent.get(url_for(state))
+        if exist_contacts_of_state?
+          @table_contains_contact = @page.at('#ctl00_cphDefaultPage_tabContainer_sheetDetails_cricosContactDetails_pnlContactLists table').children
+
+          number_of_rows_per_contact = 18
+          start_contact_row = 3
+          end_contact_row = @table_contains_contact.count - number_of_rows_per_contact
+
+          for i in (start_contact_row..end_contact_row).step(number_of_rows_per_contact)
+            @row_index = i
+
+            contact                = Contact.new
+            contact.type_of_course = find_type_of_course
+            contact.name           = find_name
+            contact.organisation   = find_organisation
+            contact.postal_address = find_postal_address
+            contact.telephone      = find_telephone
+            contact.facsimile      = find_facsimile
+            contact.email          = find_email
+
+            contacts << contact
+          end
+        end
+      end
+
+      contacts
+    end
+
+    private
+
+    attr_reader :agent, :page
+
+    def url_for(state_code)
+      "#{CONTACT_URL}?StateCode=#{state_code}"
+    end
+
+    def exist_contacts_of_state?
+      !!@page.at('#__tab_ctl00_cphDefaultPage_tabContainer_sheetDetails')
+    end
+
+    def find_value_of_field(field)
+      field.nil? ? nil : field.text.strip
+    end
+
+    def find_type_of_course
+      find_value_of_field(@table_contains_contact[@row_index])
+    end
+
+    def find_name
+      name_row = @table_contains_contact[@row_index+4].children
+      find_value_of_field(name_row[3]).empty? ? find_value_of_field(name_row[2]) : find_value_of_field(name_row[3])
+    end
+
+    def find_organisation
+      organisation_row = @table_contains_contact[@row_index+6].children
+      find_value_of_field(organisation_row[3])
+    end
+
+    def find_postal_address
+      address = Address.new
+
+      address_row = @table_contains_contact[@row_index+8].children
+      postal_address_cell = address_row[3].children
+
+      # delete <br>
+      lines = postal_address_cell - postal_address_cell.css('br')
+      address.address_line_1 = find_value_of_field(lines[0])
+
+      if line2 = find_value_of_field(lines[1])
+        address.suburb, address.state, address.postcode = extract_suburb_and_state_and_postcode_from(line2)
+      end
+
+      address
+    end
+
+    def extract_suburb_and_state_and_postcode_from(line)
+      line.scan(/^(.*)\s(#{australia_states_code_regex})\s(#{australia_postcode_regex})$/).first
+    end
+
+    def australia_states_code_regex
+      'ACT|NSW|NT|QLD|SA|TAS|VIC|WA'
+    end
+
+    def australia_postcode_regex
+      '\d{4}'
+    end
+
+    def find_telephone
+      telephone_row = @table_contains_contact[@row_index+10].children
+      find_value_of_field(telephone_row[3])
+    end
+
+    def find_facsimile
+      facsimile_row = @table_contains_contact[@row_index+12].children
+      find_value_of_field(facsimile_row[3])
+    end
+
+    def find_email
+      email_row = @table_contains_contact[@row_index+14].children
+      find_value_of_field(email_row[3])
+    end
+  end
+end

data/lib/cricos_scrape/importer/course_importer.rb

@@ -0,0 +1,291 @@
+require 'cricos_scrape/entities/course'
+require 'cricos_scrape/entities/contact_officer'
+
+module CricosScrape
+  class CourseImporter
+
+    COURSE_URL = 'http://cricos.education.gov.au/Course/CourseDetails.aspx'
+
+    def initialize(agent, **params)
+      @agent = agent
+      @course_id = params.fetch(:course_id)
+      @page = agent.get(url)
+    end
+
+    def run
+      return if course_not_found?
+
+      course                    = Course.new
+      course.course_id          = course_id
+      course.course_name        = find_course_name
+      course.course_code        = find_course_code
+      course.dual_qualification = find_dual_qualification
+      course.field_of_education = find_field_of_education
+      course.broad_field        = find_education_broad_field
+      course.narrow_field       = find_education_narrow_field
+      course.detailed_field     = find_education_detailed_field
+      course.course_level       = find_course_level
+      course.foundation_studies = find_foundation_studies
+      course.work_component     = find_work_component
+      course.course_language    = find_course_language
+      course.duration           = find_duration
+      course.total_cost         = find_total_cost
+
+      course.contact_officers   = find_contact_officers
+      course.location_ids       = find_course_location
+
+      course
+    end
+
+    private
+
+    attr_reader :agent, :course_id, :page
+
+    def url
+      "#{COURSE_URL}?CourseID=#{course_id}"
+    end
+
+    # there is no record not found page
+    # instead a search page is returned
+    def course_not_found?
+      @page.at('#contentBody h1').text == "Course Search"
+    end
+
+    def find_value_of_field(field)
+      field.text.strip unless field.nil?
+    end
+
+    def find_course_name
+      field = @page.at('#ctl00_cphDefaultPage_tabContainer_sheetCourseDetail_courseDetail_lblCourseName')
+      find_value_of_field(field)
+    end
+
+    def find_course_code
+      field = @page.at('#ctl00_cphDefaultPage_tabContainer_sheetCourseDetail_courseDetail_lblCourseCode')
+      find_value_of_field(field)
+    end
+
+    def find_dual_qualification
+      field = @page.at('#ctl00_cphDefaultPage_tabContainer_sheetCourseDetail_courseDetail_lblDualQualification')
+      find_value_of_field(field)
+    end
+
+    def find_field_of_education
+      row = @page.at('#ctl00_cphDefaultPage_tabContainer_sheetCourseDetail_courseDetail_trFofEHeader').children
+      # NOTE: A space lookalike character might be returned. This is to ensure its conversion to a correct space
+      find_value_of_field(row[3]).ord == 160 ? '' : find_value_of_field(row[3])
+    end
+
+    def find_education_broad_field
+      field = @page.at('#ctl00_cphDefaultPage_tabContainer_sheetCourseDetail_courseDetail_lblFieldOfEducationBroad1')
+      find_value_of_field(field)
+    end
+
+    def find_education_narrow_field
+      field = @page.at('#ctl00_cphDefaultPage_tabContainer_sheetCourseDetail_courseDetail_lblFieldOfEducationNarrow1')
+      find_value_of_field(field)
+    end
+
+    def find_education_detailed_field
+      field = @page.at('#ctl00_cphDefaultPage_tabContainer_sheetCourseDetail_courseDetail_lblFieldOfEducationDetailed1')
+      find_value_of_field(field)
+    end
+
+    def find_course_level
+      field = @page.at('#ctl00_cphDefaultPage_tabContainer_sheetCourseDetail_courseDetail_lblCourseLevel')
+      find_value_of_field(field)
+    end
+
+    def find_foundation_studies
+      field = @page.at('#ctl00_cphDefaultPage_tabContainer_sheetCourseDetail_courseDetail_lblFoundationStudies')
+      find_value_of_field(field)
+    end
+
+    def find_work_component
+      field = @page.at('#ctl00_cphDefaultPage_tabContainer_sheetCourseDetail_courseDetail_lblWorkComponent')
+      find_value_of_field(field)
+    end
+
+    def find_course_language
+      field = @page.at('#ctl00_cphDefaultPage_tabContainer_sheetCourseDetail_courseDetail_lblCourseLanguage')
+      find_value_of_field(field)
+    end
+
+    def find_duration
+      field = @page.at('#ctl00_cphDefaultPage_tabContainer_sheetCourseDetail_courseDetail_lblDuration')
+      find_value_of_field(field)
+    end
+
+    def find_total_cost
+      field = @page.at('#ctl00_cphDefaultPage_tabContainer_sheetCourseDetail_courseDetail_lblTotalCourseCost')
+      find_value_of_field(field)
+    end
+
+    def find_contact_officers
+      contact_officers = []
+
+      contact_officers_list = @page.search('//div[starts-with(@id, "ctl00_cphDefaultPage_tabContainer_sheetContactDetail_contactDetail_pnl")]')
+
+      contact_officers_list.each do |contact_officer|
+        @contact_officer_area = contact_officer
+        @contact_officer_table = @contact_officer_area.at('table').children
+
+        if contains_contact_details_grid?
+          contact_officers += find_contact_officer_grid
+        else
+          contact_officers << find_contact_officer
+        end
+      end
+
+      contact_officers
+    end
+
+    def find_contact_officer_grid
+      contact_officers = []
+
+      excess_row_at_the_end_table = 2
+      data_row_start = 3
+      data_row_end = @contact_officer_table.count - excess_row_at_the_end_table
+
+      for i in data_row_start..data_row_end
+        contact_row = @contact_officer_table[i].children
+
+        contact = ContactOfficer.new
+        contact.role      = find_contact_officer_role
+        contact.name      = find_value_of_field(contact_row[1])
+        contact.phone     = find_value_of_field(contact_row[2])
+        contact.fax       = find_value_of_field(contact_row[3])
+        contact.email     = find_value_of_field(contact_row[4])
+
+        contact_officers << contact
+      end
+
+      contact_officers
+    end
+
+    def find_contact_officer
+      contact           = ContactOfficer.new
+      contact.role      = find_contact_officer_role
+      contact.name      = find_contact_officer_name
+      contact.title     = find_contact_officer_title
+      contact.phone     = find_contact_officer_phone
+      contact.fax       = find_contact_officer_fax
+      contact.email     = find_contact_officer_email
+
+      contact
+    end
+
+    def find_contact_officer_role
+      row = @contact_officer_area.children
+      find_value_of_field(row[1]).sub(':', '')
+    end
+
+    def find_contact_officer_name
+      row = @contact_officer_table[1].children
+      find_value_of_field(row[3])
+    end
+
+    def find_contact_officer_title
+      row = @contact_officer_table[3].children
+      find_value_of_field(row[3])
+    end
+
+    def find_contact_officer_phone
+      row = @contact_officer_table[5].children
+      find_value_of_field(row[3])
+    end
+
+    def find_contact_officer_fax
+      row = @contact_officer_table[7].children
+      find_value_of_field(row[3])
+    end
+
+    def find_contact_officer_email
+      row = @contact_officer_table[9]
+      find_value_of_field(row.children[3]) unless row.nil?
+    end
+
+    def contains_contact_details_grid?
+      contact_officer_area_css_id = @contact_officer_area.attributes['id'].text
+      @page.search("//*[@id='#{contact_officer_area_css_id}']/div/table[starts-with(@id, 'ctl00_cphDefaultPage_tabContainer_sheetContactDetail_contactDetail_grid')]").any?
+    end
+
+    #Get all locations of course
+    def find_course_location
+      location_ids = []
+
+      if location_results_paginated?
+        for page_number in 1..total_pages
+          jump_to_page(page_number)
+          location_ids += fetch_location_ids_from_current_page
+        end
+      else
+        location_ids += fetch_location_ids_from_current_page
+      end
+
+      location_ids
+    end
+
+    def pagination
+      @page.at('#ctl00_cphDefaultPage_tabContainer_sheetCourseDetail_courseLocationList_gridSearchResults .gridPager')
+    end
+
+    def location_results_paginated?
+      !!pagination
+    end
+
+    def total_pages
+      pagination.children[1].text.strip[/^Page [0-9]+ of ([0-9]+).*/, 1].to_i
+    end
+
+    def current_pagination_page
+      pagination.children[1].text.strip[/^Page ([0-9]+) of [0-9]+.*/, 1].to_i
+    end
+
+    def jump_to_page(page_number)
+      return @page if page_number == current_pagination_page
+
+      hidden_form = @page.form_with :id => "aspnetForm"
+      hidden_form['__EVENTTARGET'] = 'ctl00$cphDefaultPage$tabContainer$sheetCourseDetail$courseLocationList$gridSearchResults'
+      hidden_form['__EVENTARGUMENT'] = "Page$#{page_number}"
+      begin
+        @page = hidden_form.submit(nil, {'action' => 'change-page'})
+      rescue Mechanize::ResponseCodeError
+        sleep 5
+        scrape_course(course_id)
+      end
+    end
+
+    def get_location_id(row_index)
+      hidden_form = @page.form_with :id => "aspnetForm"
+      hidden_form['__EVENTTARGET'] = 'ctl00$cphDefaultPage$tabContainer$sheetCourseDetail$courseLocationList$gridSearchResults'
+      hidden_form['__EVENTARGUMENT'] = "click-#{row_index-3}"
+      begin
+        course_page = hidden_form.submit(nil, {'action' => 'get-location-id'})
+      rescue Mechanize::ResponseCodeError
+        sleep 5
+        scrape_course(course_id)
+      end
+
+      course_page.uri.to_s[/LocationID=([0-9]+)/, 1]
+    end
+
+    def fetch_location_ids_from_current_page
+      location_ids = []
+
+      # location_list is table contains locations in current page
+      location_list = @page.at('#ctl00_cphDefaultPage_tabContainer_sheetCourseDetail_courseLocationList_gridSearchResults').children
+
+      excess_row_at_the_end_table = location_results_paginated? ? 3 : 2
+      start_location_row = 3
+      end_location_row = location_list.count - excess_row_at_the_end_table
+
+      for i in start_location_row..end_location_row
+        location_ids << get_location_id(i)
+      end
+
+      location_ids
+    end
+
+  end
+end