mwcrawler 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 535ef5f765febcd8e34046c5913006148e354e4c9d4735709ad4bba9fc30b058
+  data.tar.gz: f8014655d47a46e88bfeeccca72557341701685402182e6b0b878abfd362e534
+SHA512:
+  metadata.gz: 9a2855da03d0feaa6386112e4ede645a5638a2f1d68628f115012220c81d4672959c358bd496853e1a10ce473a493e29cb0f2012803f4c484677ebd4df36b738
+  data.tar.gz: 34b27e7cbd83e191fb0065ba80b1ee62adcf7086a77e6e690e3fc426d958584420ed7dad110d18a55b402a1fba5408b3895399e7b0c38745332c9dc06fcd2594

data/.gitignore

@@ -0,0 +1,14 @@
+/.bundle/
+/.yardoc
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/
+/.vscode
+*.gem
+
+# rspec failure tracking
+.rspec_status
+*.json

data/.rspec

@@ -0,0 +1,2 @@
+--color
+--require spec_helper

data/.rubocop.yml

@@ -0,0 +1,11 @@
+AllCops:
+  Exclude:
+    - vendor/bundle/**/*
+    - '*.gemspec'
+
+Metrics/LineLength:
+  Max: 120
+
+Metrics/BlockLength:
+  Exclude:
+    - 'spec/**/*'

data/.ruby-version

@@ -0,0 +1 @@
+ruby-2.5.1

data/.travis.yml

@@ -0,0 +1,11 @@
+os: linux
+rvm: 2.5.1
+dist: trusty
+sudo: false
+cache: bundler
+before_install:
+  - gem install bundler -v 1.16.1 --no-document
+  - bundle install
+script:
+  - bundle exec rubocop
+  - bundle exec rspec

data/Gemfile

@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+source 'https://rubygems.org'
+
+git_source(:github) { |repo_name| "https://github.com/#{repo_name}" }
+
+# Specify your gem's dependencies in mwcrawler.gemspec
+gemspec

data/Gemfile.lock

@@ -0,0 +1,95 @@
+PATH
+  remote: .
+  specs:
+    mwcrawler (0.1.0)
+      nokogiri (~> 1.8)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    addressable (2.5.2)
+      public_suffix (>= 2.0.2, < 4.0)
+    ast (2.4.0)
+    coderay (1.1.2)
+    coveralls (0.8.22)
+      json (>= 1.8, < 3)
+      simplecov (~> 0.16.1)
+      term-ansicolor (~> 1.3)
+      thor (~> 0.19.4)
+      tins (~> 1.6)
+    crack (0.4.3)
+      safe_yaml (~> 1.0.0)
+    diff-lcs (1.3)
+    docile (1.3.1)
+    hashdiff (0.3.7)
+    jaro_winkler (1.5.1)
+    json (2.1.0)
+    method_source (0.9.0)
+    mini_portile2 (2.4.0)
+    nokogiri (1.10.9)
+      mini_portile2 (~> 2.4.0)
+    parallel (1.12.1)
+    parser (2.5.1.2)
+      ast (~> 2.4.0)
+    powerpack (0.1.2)
+    pry (0.11.3)
+      coderay (~> 1.1.0)
+      method_source (~> 0.9.0)
+    public_suffix (3.0.3)
+    rainbow (3.0.0)
+    rake (13.0.1)
+    rspec (3.8.0)
+      rspec-core (~> 3.8.0)
+      rspec-expectations (~> 3.8.0)
+      rspec-mocks (~> 3.8.0)
+    rspec-core (3.8.0)
+      rspec-support (~> 3.8.0)
+    rspec-expectations (3.8.1)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.8.0)
+    rspec-mocks (3.8.0)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.8.0)
+    rspec-support (3.8.0)
+    rubocop (0.59.2)
+      jaro_winkler (~> 1.5.1)
+      parallel (~> 1.10)
+      parser (>= 2.5, != 2.5.1.1)
+      powerpack (~> 0.1)
+      rainbow (>= 2.2.2, < 4.0)
+      ruby-progressbar (~> 1.7)
+      unicode-display_width (~> 1.0, >= 1.0.1)
+    ruby-progressbar (1.10.0)
+    safe_yaml (1.0.4)
+    simplecov (0.16.1)
+      docile (~> 1.1)
+      json (>= 1.8, < 3)
+      simplecov-html (~> 0.10.0)
+    simplecov-html (0.10.2)
+    term-ansicolor (1.7.0)
+      tins (~> 1.0)
+    thor (0.19.4)
+    tins (1.18.0)
+    unicode-display_width (1.4.0)
+    vcr (4.0.0)
+    webmock (3.4.2)
+      addressable (>= 2.3.6)
+      crack (>= 0.3.2)
+      hashdiff
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  bundler (~> 1.16)
+  coveralls
+  mwcrawler!
+  pry (~> 0.11)
+  rake (~> 13.0)
+  rspec (~> 3.0)
+  rubocop (~> 0.59.2)
+  vcr (~> 4.0)
+  webmock (~> 3.4)
+
+BUNDLED WITH
+   1.16.6

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2018 vitor pontes
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,86 @@
+# Mwcrawler
+
+Mwcrawler is a gem for parsing UnB's Matricula Web data into consumable hashes.
+
+[![Build Status](https://travis-ci.com/danilodelyima/mwcrawler.svg?branch=master)](https://travis-ci.com/danilodelyima/mwcrawler)
+[![Coverage Status](https://coveralls.io/repos/github/danilodelyima/mwcrawler/badge.svg?branch=master)](https://coveralls.io/github/danilodelyima/mwcrawler?branch=master)
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'mwcrawler'
+```
+
+And then execute:
+
+    bundle
+
+Or install it yourself as:
+
+    gem install mwcrawler
+
+## Usage
+
+First instantiate a new crawler `crawler = Mwcrawler::Crawler.new` then you can crawl like so:
+
+```ruby
+courses_hash = crawler.courses
+# return example
+[{"type"=>"Presencial",
+  "code"=>"19",
+  "name"=>"ADMINISTRAÇÃO",
+  "shift"=>"Diurno",
+  "curriculums"=>
+   [{"name"=>"Administração",
+     "degree"=>"Bacharel",
+     "semester_max"=>"8",
+     "semester_min"=>"16",
+     "credits"=>"200"}]},
+ {"type"=>"Presencial",
+  "code"=>"701",
+  "name"=>"ADMINISTRAÇÃO",
+  "shift"=>"Noturno",
+  "curriculums"=>
+   [{"name"=>"Administração",
+     "degree"=>"Bacharel",
+     "semester_max"=>"8",
+     "semester_min"=>"16",
+     "credits"=>"200"}]}
+]
+```
+
+The crawled campus by default is `:darcy_ribeiro` campus,
+but you can specify another `crawler.classes(:planaltina)`.
+
+The available resources are:
+
+- `classes`
+- `courses`
+- `departments`
+- `curriculum`
+
+While `classes` and `curriculum` take `course_code` as param for crawling, `courses` and `departments` take as params any of the four campuses `:darcy_ribeiro`, `:planaltina`, `:ceilandia` and `:gama`.
+
+The utility method `semester` returns the current semester.
+
+## Development
+
+After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
+
+To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at https://github.com/danilodelyima/mwcrawler.
+
+# Guidelines
+
+When developing new features the interface must reflect how much scrapping is necessary. In other
+words, if many pages are crawled the user must call many methods. This way we don't overload method
+with functionalities and the user developer can grasp more easily the cost of scrapping that info.
+
+## License
+
+The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT).

data/Rakefile

@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+require 'bundler/gem_tasks'
+require 'rspec/core/rake_task'
+
+RSpec::Core::RakeTask.new(:spec)
+
+task default: :spec

data/TOTAL.txt

@@ -0,0 +1,14 @@
+2017-2
+
+Total de turmas do Darcy: 7366
+Total de turmas Planaltina: 284
+Total de turmas Ceilândia: 526
+Total de turmas Gama: 480
+
+
+2018-1
+
+Total de turmas do Darcy: 9779 -- 34 min
+Total de turmas Planaltina: 288 -- 2 min
+Total de turmas Ceilândia: 543 -- 3 min
+Total de turmas Gama: 298 -- 2 min

data/bin/console

@@ -0,0 +1,12 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+require 'bundler/setup'
+require 'mwcrawler'
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+require 'pry'
+Pry.start

data/bin/setup

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+set -vx
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/lib/mwcrawler.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+require 'mwcrawler/version'
+require 'nokogiri'
+require 'pry'
+require 'open-uri'
+require 'json'
+
+require 'mwcrawler/classes'
+require 'mwcrawler/courses'
+require 'mwcrawler/departments'
+require 'mwcrawler/subjects'
+require 'mwcrawler/helpers'
+require 'mwcrawler/crawler'
+
+module Mwcrawler
+  # DOMINIO
+  SITE = 'https://matriculaweb.unb.br/'
+end

data/lib/mwcrawler/classes.rb

@@ -0,0 +1,92 @@
+# frozen_string_literal: true
+
+module Mwcrawler
+  # Scraps Classes by campus
+  module Classes
+    def self.scrap(department_code)
+      courses_links = scrap_courses_links(department_code)
+      rows = []
+      courses_links.each do |course_link|
+        rows += scrap_classes(course_link)
+      end
+      rows
+    end
+
+    private_class_method def self.scrap_courses_links(department_code)
+      page = Helpers.set_crawler(department_code, 'graduacao/oferta_dis.aspx?cod=', exact: true)
+      page.css('#datatable tr td:nth-child(2) a')
+          .map { |link| link['href'] }
+    end
+
+    private_class_method def self.scrap_classes(course_link)
+      rows = []
+
+      page = Helpers.set_crawler(course_link, 'graduacao/', exact: true)
+      page_classes = page.css('.tabela-oferta .turma').map(&:text)
+
+      page_classes.each_with_index do |cl, i|
+        row_init = class_row_init(page, cl)
+        rows << scrap_row(row_init, page, i)
+        Helpers.log "Total de turmas: #{rows.size}"
+      end
+      rows
+    end
+    private_class_method def self.class_row_init(page, name)
+      { department: page.css('#datatable tr:first-child a').text,
+        code:  page.css('#datatable')[0].css('tr:nth-child(2) td').text.to_i,
+        course_code: scrap_course_code(page),
+        credits: scrap_credit_hash(page),
+        name: name }
+    end
+
+    private_class_method def self.scrap_course_code(page)
+      course_uri = page.css('#datatable')[0].css('tr:nth-child(3) td a').first['href']
+      Helpers.uri_query_params(course_uri)['cod'].to_i
+    end
+
+    private_class_method def self.scrap_credit_hash(page)
+      credit_string = page.css('#datatable')[0].css('tr:nth-child(4) td').text
+      credits = credit_string.split('-').map(&:to_i)
+      { theory: credits[0], practical: credits[1], extension: credits[2], study: credits[3] }
+    end
+
+    private_class_method def self.scrap_row(row_init, page, count)
+      row = row_init
+      row.merge(scrap_vacancies(page, count))
+      # HORARIOS
+      row[:schedules] = scrap_schedules(page, count)
+      # PROFESSORES
+      row[:teachers] = scrap_teachers(page, count)
+      row
+    end
+
+    private_class_method def self.scrap_schedules(page, count)
+      schedules = page.css('.tabela-oferta')[count]
+                  .css('tr td:nth-child(4) .table')
+                  .css('td').map(&:text)
+
+      Helpers.format_hours(schedules)
+    end
+
+    private_class_method def self.scrap_teachers(page, count)
+      teachers = page.css('.tabela-oferta')[count]
+                 .css('tr td:nth-child(5) td')
+                     .map(&:text)
+
+      Helpers.format_teachers(teachers)
+    end
+
+    private_class_method def self.scrap_vacancies(page, count)
+      {
+        vacancies_total: scrap_vacancy(1, page, count),
+        vacancies_occupied: scrap_vacancy(2, page, count),
+        vacancies_free: scrap_vacancy(3, page, count)
+      }
+    end
+
+    private_class_method def self.scrap_vacancy(vacancy_row, page, count)
+      page.css('.tabela-oferta')[count]
+          .css(".tabela-vagas tr:nth-child(#{vacancy_row}) td:nth-child(3)").text
+    end
+  end
+end

data/lib/mwcrawler/courses.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+require 'mwcrawler/curriculum'
+
+module Mwcrawler
+  # Scraps Courses by campus
+  module Courses
+    def self.scrap(campus)
+      page = Helpers.set_crawler(campus, 'graduacao/curso_rel.aspx?cod=')
+      courses = page.css('#datatable tr td').map(&:text)
+
+      # CADA CURSO SERA UMA LINHA, ENTAO rows E O CONJUNTO DE TODAS AS TURMAS
+      rows = []
+      rows << scrap_row(courses) until courses.empty?
+      Helpers.log "Total de cursos: #{rows.count}"
+
+      rows
+    end
+
+    private_class_method def self.scrap_row(courses)
+      row = {}
+      row['type'] = courses.shift
+      row['code'] = courses.shift
+      row['name'] = courses.shift
+      row['shift'] = courses.shift
+      row['curriculums'] = Curriculum.scrap(row['code'])
+      row
+    end
+  end
+end

data/lib/mwcrawler/crawler.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+module Mwcrawler
+  # Main api for crawling
+  class Crawler
+    include Mwcrawler
+
+    SCRAPPERS = {
+      courses: Courses,
+      classes: Classes,
+      departments: Departments
+    }.freeze
+
+    SCRAPPERS.keys.each do |method|
+      define_method(method) do |campus = :darcy_ribeiro, options = { log: false }|
+        Options.init(options)
+        SCRAPPERS[method].scrap campus
+      end
+    end
+
+    def subjects(department, options = { log: false })
+      Options.init(options)
+      Subjects.scrap department, options
+    end
+
+    def semester
+      page = Helpers.set_crawler(nil, 'graduacao/default.aspx', exact: true)
+      page.css("a[title='Período Atual'] span").first.text
+    end
+  end
+end

data/lib/mwcrawler/curriculum.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+module Mwcrawler
+  # Scraps curriculums by course code
+  module Curriculum
+    def self.scrap(code)
+      rows = []
+      page = Helpers.set_crawler(code, 'graduacao/curso_dados.aspx?cod=', exact: true)
+      curriculums = page.css('.table-responsive h4').map { |item| item.children[0].text }
+      page.css('.table-responsive .table').each do |table|
+        rows << scrap_row(curriculums.shift, table)
+      end
+      rows
+    end
+
+    private_class_method def self.scrap_row(curriculum_name, table)
+      row = {}
+      row['name'] = curriculum_name
+      row['degree'] = table.css('tr:first td').text
+      row['semester_max'] = table.css('tr:nth-child(2) td').text
+      row['semester_min'] = table.css('tr:nth-child(3) td').text
+      row['credits'] = table.css('tr:nth-child(4) td').text
+      row
+    end
+  end
+end

data/lib/mwcrawler/departments.rb

@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+module Mwcrawler
+  # Scraps Departments by campus
+  module Departments
+    def self.scrap(campus)
+      page = Helpers.set_crawler(campus, 'graduacao/oferta_dep.aspx?cod=')
+      departments = page.css('#datatable tr td').map(&:text)
+
+      # CADA DEPARTAMENTO SERA UMA LINHA, ENTAO rows E O CONJUNTO DE TODOS OS DEPARTAMENTOS
+      rows = []
+      rows << scrap_row(departments) until departments.empty?
+      rows
+    end
+
+    private_class_method def self.scrap_row(departments)
+      row = {}
+      row['code'] = departments.shift
+      row['acronym'] = departments.shift
+      row['name'] = departments.shift
+      row
+    end
+  end
+end

data/lib/mwcrawler/helpers.rb

@@ -0,0 +1,76 @@
+# frozen_string_literal: true
+
+module Mwcrawler
+  # Controls available campuses
+  class Campuses
+    CAMPUSES = {
+      darcy_ribeiro: 1,
+      planaltina: 2,
+      ceilandia: 3,
+      gama: 4
+    }.freeze
+
+    def self.id(campus)
+      raise ArgumentError, "Campus: #{campus} not in: #{CAMPUSES.keys}" unless CAMPUSES.include? campus
+
+      CAMPUSES[campus]
+    end
+  end
+
+  # Options module
+  module Options
+    module_function
+
+    @log = false
+
+    def init(options = { log: false })
+      @log = options[:log].freeze
+    end
+
+    def log_enabled?
+      @log
+    end
+  end
+
+  # Helper methods used throughout the lib
+  class Helpers
+    def self.format_hours(schedules, row = [])
+      until schedules.empty?
+        schedule = []
+        schedule << schedules.shift # DIA
+        schedule << schedules.shift # HORARIO DE INICIO
+        schedule << schedules.shift # HORARIO DE FIM
+        schedules.shift # RETIRANDO LIXO
+        schedule << schedules.shift # LOCAL DA AULA
+        row << schedule
+      end
+      row
+    end
+
+    def self.format_teachers(teachers)
+      teachers.empty? ? ['A Designar'] : teachers
+    end
+
+    # MODE: TURMAS, CURSOS OU CURRICULO
+    def self.set_crawler(id, search_mode, options = { exact: false })
+      id = Campuses.id id unless options[:exact]
+      url = SITE + search_mode + id.to_s
+      Nokogiri::HTML(URI.parse(url).open)
+    end
+
+    def self.write_json(file_name, object)
+      File.open(file_name, 'w+') do |f|
+        f.write object.to_json
+      end
+    end
+
+    def self.log(msg)
+      puts msg if Options.log_enabled?
+    end
+
+    def self.uri_query_params(uri)
+      query_string = URI.parse(uri).query
+      query_string.split('&').map { |param| param.split('=') }.to_h
+    end
+  end
+end

data/lib/mwcrawler/subjects.rb

@@ -0,0 +1,49 @@
+# frozen_string_literal: true
+
+module Mwcrawler
+  # Scraps Subjects by department
+  module Subjects
+    def self.scrap(department_or_id, options)
+      if options[:by_id]
+        subject_by_id(department_or_id)
+      elsif options[:by_department]
+        subject_by_department(department_or_id)
+      else
+        raise ArgumentError, 'second argument not specified. You can find a subject by department code or id'
+      end
+    end
+
+    private_class_method def self.subject_by_department(department)
+      page = Helpers.set_crawler(department, 'graduacao/oferta_dis.aspx?cod=', exact: true)
+      scrap_row(department, page)
+    end
+
+    private_class_method def self.subject_by_id(id)
+      page = Helpers.set_crawler(id, 'graduacao/oferta_dados.aspx?cod=', exact: true)
+      row_init_by_id(page)
+    end
+
+    private_class_method def self.row_init_by_id(page)
+      { code: page.css('#datatable')[0].css('tr:nth-child(2) td').text.to_i,
+        name: page.css('#datatable')[0].css('tr:nth-child(3) td').text,
+        department: page.css('#datatable tr:first-child a').first['href'].scan(/\d+/)[0].to_i,
+        level: 'graduação' }
+    end
+
+    private_class_method def self.scrap_row(dep_code, page)
+      subjects = []
+      length = page.css('#datatable tr td:nth-child(1)').count
+      length.times do |i|
+        subjects << row_init_by_department(page, dep_code, i)
+      end
+      subjects
+    end
+
+    private_class_method def self.row_init_by_department(page, dep_code, index)
+      { code: page.css('#datatable tr td:nth-child(1)').map(&:text)[index].to_i,
+        name: page.css('#datatable tr td:nth-child(2)').map(&:text)[index],
+        department: dep_code.to_i,
+        level: 'graduação' }
+    end
+  end
+end

data/lib/mwcrawler/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module Mwcrawler
+  VERSION = '0.1.0'
+end

data/mwcrawler.gemspec

@@ -0,0 +1,41 @@
+lib = File.expand_path('lib', __dir__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'mwcrawler/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = 'mwcrawler'
+  spec.version       = Mwcrawler::VERSION
+  spec.authors       = ['Danilo de Lima', 'vitor pontes']
+  spec.email         = ['vitormax2005@hotmail.com']
+
+  spec.summary       = 'Gema para webscrapping do sistemas de matriculas da unb Matricula Web.'
+  spec.description   = 'Essa gema provê uma api ruby para se fazer o scrapping de páginas html do sistema matricula web e retornar um conteudo que pode ser mais facilmente processado pelo programa'
+  spec.homepage      = 'https://github.com/danilodelyima/mwcrawler'
+  spec.license       = 'MIT'
+
+  # Prevent pushing this gem to RubyGems.org. To allow pushes either set the 'allowed_push_host'
+  # to allow pushing to a single host or delete this section to allow pushing to any host.
+  # if spec.respond_to?(:metadata)
+  #   spec.metadata['allowed_push_host'] = "TODO: Set to 'http://mygemserver.com'"
+  # else
+  #   raise 'RubyGems 2.0 or newer is required to protect against ' \
+  #     'public gem pushes.'
+  # end
+
+  spec.files = `git ls-files -z`.split("\x0").reject do |f|
+    f.match(%r{^(test|spec|features)/})
+  end
+  spec.bindir        = 'exe'
+  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.require_paths = ['lib']
+
+  spec.add_development_dependency 'bundler', '~> 1.16'
+  spec.add_development_dependency 'coveralls'
+  spec.add_development_dependency 'pry', '~> 0.11'
+  spec.add_development_dependency 'rake', '~> 13.0'
+  spec.add_development_dependency 'rspec', '~> 3.0'
+  spec.add_development_dependency 'vcr', '~> 4.0'
+  spec.add_development_dependency 'webmock', '~>  3.4'
+  spec.add_development_dependency 'rubocop', '~> 0.59.2'
+  spec.add_dependency 'nokogiri', '~> 1.8'
+end

metadata

@@ -0,0 +1,196 @@
+--- !ruby/object:Gem::Specification
+name: mwcrawler
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Danilo de Lima
+- vitor pontes
+autorequire: 
+bindir: exe
+cert_chain: []
+date: 2020-05-29 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.16'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.16'
+- !ruby/object:Gem::Dependency
+  name: coveralls
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: pry
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.11'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.11'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '13.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '13.0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+- !ruby/object:Gem::Dependency
+  name: vcr
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '4.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '4.0'
+- !ruby/object:Gem::Dependency
+  name: webmock
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.4'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.4'
+- !ruby/object:Gem::Dependency
+  name: rubocop
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.59.2
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.59.2
+- !ruby/object:Gem::Dependency
+  name: nokogiri
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.8'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.8'
+description: Essa gema provê uma api ruby para se fazer o scrapping de páginas html
+  do sistema matricula web e retornar um conteudo que pode ser mais facilmente processado
+  pelo programa
+email:
+- vitormax2005@hotmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".rspec"
+- ".rubocop.yml"
+- ".ruby-version"
+- ".travis.yml"
+- Gemfile
+- Gemfile.lock
+- LICENSE.txt
+- README.md
+- Rakefile
+- TOTAL.txt
+- bin/console
+- bin/setup
+- lib/mwcrawler.rb
+- lib/mwcrawler/classes.rb
+- lib/mwcrawler/courses.rb
+- lib/mwcrawler/crawler.rb
+- lib/mwcrawler/curriculum.rb
+- lib/mwcrawler/departments.rb
+- lib/mwcrawler/helpers.rb
+- lib/mwcrawler/subjects.rb
+- lib/mwcrawler/version.rb
+- mwcrawler.gemspec
+homepage: https://github.com/danilodelyima/mwcrawler
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.7.6
+signing_key: 
+specification_version: 4
+summary: Gema para webscrapping do sistemas de matriculas da unb Matricula Web.
+test_files: []