scrap_cbf 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 468506451d0e8869e39a079ddaacf33a8aacc1e2a79052cdb251611217874c0d
+  data.tar.gz: 05a8c5cc7e71b8323ee4e3bd73ae723519a24ffe357e392910c044186496b586
+SHA512:
+  metadata.gz: ff2bad7d68f58a68d8499c34f175bac2231616f41c99190fa91f80e0a8ed1ecf59fe7717a81bc3a86cefe73fdefaa52028248a6f5f7456a4f40ad85ef1e468b4
+  data.tar.gz: 01bc568cdca4a5ccd530bdafd42706b45724f5cd1f6bb01db23182ea8f9ea74eba4e031f7c0603fcefadfba900b6643616486e8499c354c13ba578760096da83

data/lib/scrap_cbf.rb

@@ -0,0 +1,92 @@
+# frozen_string_literal: true
+
+require 'nokogiri'
+require 'open-uri'
+require 'json'
+require 'active_support/core_ext/hash/indifferent_access'
+require 'forwardable'
+
+require_relative 'scrap_cbf/errors'
+require_relative 'scrap_cbf/formattable'
+require_relative 'scrap_cbf/printable'
+require_relative 'scrap_cbf/document'
+require_relative 'scrap_cbf/models/table/header_column'
+require_relative 'scrap_cbf/models/table/row'
+require_relative 'scrap_cbf/models/table/cell'
+require_relative 'scrap_cbf/models/championship'
+require_relative 'scrap_cbf/models/team'
+require_relative 'scrap_cbf/models/round'
+require_relative 'scrap_cbf/models/match'
+require_relative 'scrap_cbf/models/ranking'
+require_relative 'scrap_cbf/helpers/matches_helper'
+require_relative 'scrap_cbf/helpers/rankings_helper'
+require_relative 'scrap_cbf/helpers/teams_helper'
+require_relative 'scrap_cbf/builders/matches_builder'
+require_relative 'scrap_cbf/builders/matches_per_round_builder'
+require_relative 'scrap_cbf/builders/rankings_builder'
+require_relative 'scrap_cbf/builders/rounds_builder'
+require_relative 'scrap_cbf/builders/teams_builder'
+
+# ScrapCbf is a gem created for scraping data from the CBF official page.
+# Some of the data found on the CBF page are:
+# teams, matches, rounds and ranking table from all championships founded
+# on the official page.
+class ScrapCbf
+  include Formattable
+  include Printable
+
+  # @!attribute [r] document
+  #   @return [ScrapCbf::Document] ScrapCbf::Document instance.
+  # @!attribute [r] championship
+  #   @return [ScrapCbf::Championship] ScrapCbf::Championship instance.
+  attr_reader :document
+  attr_reader :championship
+
+  # @param [Hash] opts
+  # @option opts [Integer] :year The Championship year.
+  # @option opts [Symbol] :serie The Championship serie.
+  # @option opts [Symbol] :load_from_sample Load championship from sample.
+  # @option opts [Symbol] :sample_path to the sample otherwise default
+  #
+  # @return [ScrapCbf] new instance
+  def initialize(opts = {})
+    year = opts.fetch(:year, Date.today.year.to_i)
+    serie = opts.fetch(:serie, :serie_a)
+
+    @document = Document.new(year, serie, opts)
+    @parsed_document = @document.parsed_document
+    @championship = Championship.new(year, serie)
+  end
+
+  # returns all entities scraped in hash format.
+  def to_h
+    {
+      championship: championship.to_h,
+      matches: matches.to_h,
+      rankings: rankings.to_h,
+      rounds: rounds.to_h,
+      teams: teams.to_h
+
+    }.with_indifferent_access
+  end
+
+  # @return [MatchesBuilder] instance.
+  def matches
+    @matches ||= rounds.matches_builder
+  end
+
+  # @return [RankingsBuilder] instance.
+  def rankings
+    @rankings ||= RankingsBuilder.new(@parsed_document, @championship)
+  end
+
+  # @return [RoundsBuilder] instance.
+  def rounds
+    @rounds ||= RoundsBuilder.new(@parsed_document, @championship)
+  end
+
+  # @return [TeamsBuilder] instance.
+  def teams
+    @teams ||= TeamsBuilder.new(@parsed_document)
+  end
+end

data/lib/scrap_cbf/builders/matches_builder.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+class ScrapCbf
+  class MatchesBuilder
+    extend Forwardable
+    include Formattable
+    include Printable
+
+    delegate %i[each map] => :@matches
+
+    attr_accessor :matches
+
+    def initialize(matches)
+      @matches = matches
+    end
+
+    def to_h
+      @matches.map(&:to_h)
+    end
+  end
+end

data/lib/scrap_cbf/builders/matches_per_round_builder.rb

@@ -0,0 +1,113 @@
+# frozen_string_literal: true
+
+class ScrapCbf
+  class MatchesPerRoundBuilder
+    extend Forwardable
+    include MatchesHelper
+    include Formattable
+    include Printable
+
+    delegate %i[each map] => :@matches
+
+    attr_reader :matches
+    alias all matches
+
+    def initialize(matches_elements, round_number, championship)
+      @championship = championship
+      @matches = []
+
+      scrap_matches(matches_elements, round_number)
+    end
+
+    def to_h
+      @matches.map(&:to_h)
+    end
+
+    private
+
+    def scrap_matches(matches_elements, round_number)
+      matches_elements.children.each do |match_element|
+        next unless match_element.element?
+
+        @matches << scrap_match(match_element, round_number)
+      end
+    end
+
+    def scrap_match(match_element, round_number)
+      match = Match.new
+      match.championship = @championship.year
+      match.serie = @championship.serie
+
+      match.round = round_number
+
+      # e.g "Qua, 03/02/2021 16:00 - Jogo: 336"
+      scrap_info(match, match_element)
+      # e.g <img title="team-name">
+      scrap_teams(match, match_element)
+      # e.g "1 alteração" (can be undefined)
+      scrap_update(match, match_element)
+      # e.g "16:00" (can be found in two places, we take only the first)
+      scrap_start_at(match, match_element)
+      # e.g "1 x 1" (can be undefined)
+      scrap_score(match, match_element)
+      # e.g "Vila Belmiro - Santos - SP" (can be undefined)
+      scrap_place(match, match_element)
+
+      match
+    end
+
+    def scrap_info(match, match_element)
+      info = find_info_helper(match_element)
+      return unless info
+
+      # e.g "Jogo: 336" (always defined)
+      match.id_match = info[/Jogo: \d{1,3}$/i].gsub(/^Jogo: /, '')
+      # e.g "03/02/2021" (can be undefined)
+      match.date = info[%r{\d{2}/\d{2}/\d{2,4}}i]
+      # e.g "16:00" (can be undefined)
+      match.start_at = info[/\d{2}:\d{2}/i]
+    end
+
+    def scrap_place(match, match_element)
+      match.place = find_place_helper(match_element)
+    end
+
+    def scrap_score(match, match_element)
+      score = find_score_helper(match_element)
+
+      match.team_score = score_by_team_helper(:team, score)
+      match.opponent_score = score_by_team_helper(:opponent, score)
+    end
+
+    def scrap_update(match, match_element)
+      match.updates = find_updates_helper(match_element)
+    end
+
+    def scrap_teams(match, match_element)
+      teams_elements = match_element.css('img')
+
+      unless teams_elements.length == 2
+        raise InvalidNumberOfEntitiesError.new(:team, teams_elements.length)
+      end
+
+      teams_name = teams_elements.map do |team_element|
+        next unless team_element.element?
+
+        scrap_team_names_helper(team_element)
+      end
+
+      unless teams_name.length == 2
+        raise InvalidNumberOfEntitiesError.new(:team, teams_name.length)
+      end
+
+      match.team = teams_name[0]
+      match.opponent = teams_name[1]
+    end
+
+    def scrap_start_at(match, match_element)
+      match.start_at = find_start_at_helper(match_element) unless match.start_at
+      datetime = date_with_start_at_helper(match.date, match.start_at)
+      match.date = datetime if datetime
+    end
+  end
+end

data/lib/scrap_cbf/builders/rankings_builder.rb

@@ -0,0 +1,118 @@
+# frozen_string_literal: true
+
+class ScrapCbf
+  class RankingsBuilder
+    extend Forwardable
+    include RankingsHelper
+    include Formattable
+    include Printable
+
+    delegate [:each] => :@rankings
+
+    def initialize(document, championship)
+      @championship = championship
+      @rankings = []
+      @header = []
+      @rows   = []
+
+      tables = document.css('table')
+      table = find_table_by_header(tables, Ranking::TABLE_HEADER)
+
+      return unless table
+
+      scrap_rankings(table)
+    end
+
+    def to_h
+      @rankings.map(&:to_h)
+    end
+
+    private
+
+    def scrap_rankings(table)
+      scrap_header(table)
+      scrap_body(table)
+      create_rankings_from_table
+    end
+
+    def scrap_header(table)
+      table.css('thead > tr > th').each do |th|
+        text = th.element? && remove_whitespace(th)
+        next unless text
+
+        title = title_or_nil_helper(th)
+
+        @header << HeaderColumn.new(text, title)
+      end
+    end
+
+    def scrap_body(table)
+      table.css('tbody > tr').each do |tr_element|
+        next if tr_element.element? && element_hidden?(tr_element)
+
+        row = Row.new
+        tr_element.children.each do |td_element|
+          text = td_element.element? && remove_whitespace(td_element)
+
+          next unless text
+
+          text = scrap_position_if_exist(text)
+
+          team = scrap_team_name_if_exist(td_element)
+
+          # First cell (e.g posicao: 7º and team: Fluminense)
+          if text && !text.empty? && team && !team.empty?
+            row.cells << Cell.new(text)
+            row.cells << Cell.new(team)
+          elsif team && !team.empty?
+            row.cells << Cell.new(team)
+          else
+            row.cells << Cell.new(text)
+          end
+        end
+
+        # Add 1 to header length because on first cell we scrap 2 values
+        row_length = row.cells.length
+        header_length = @header.length + 1
+        unless row_length == header_length
+          raise RowSizeError.new(row_length, header_length)
+        end
+
+        @rows << row
+      end
+    end
+
+    def scrap_position_if_exist(text)
+      if text&.match?(/^\d{1,2}º/i)
+        position = text[/^\d{1,2}º/i].strip
+        return position.delete 'º'
+      end
+
+      text
+    end
+
+    def scrap_team_name_if_exist(element)
+      title = title_or_nil_helper(element)
+
+      return unless title&.match?(/^[a-záàâãéèêíïóôõöúç\s\-]+ - [a-z]{2}$/i)
+
+      title[/^[a-záàâãéèêíïóôõöúç\s]{3,50}/i].strip
+    end
+
+    def create_rankings_from_table
+      @rows.each do |row|
+        ranking = Ranking.new
+        ranking.championship = @championship.year
+        ranking.serie = @championship.serie
+
+        attrs_rank = Ranking::ATTRS_RANK
+
+        row.cells.each_with_index do |cell, idx|
+          ranking.send "#{attrs_rank[idx]}=", cell.value
+        end
+
+        @rankings << ranking
+      end
+    end
+  end
+end

data/lib/scrap_cbf/builders/rounds_builder.rb

@@ -0,0 +1,75 @@
+# frozen_string_literal: true
+
+class ScrapCbf
+  class RoundsBuilder
+    extend Forwardable
+    include Formattable
+    include Printable
+
+    delegate [:each] => :@rounds
+
+    def initialize(document, championship)
+      @championship = championship
+      @rounds = []
+
+      scrap_rounds(document)
+    end
+
+    def matches_builder
+      matches = @rounds.reduce([]) do |arr, round|
+        matches_per_round = round.matches
+        arr.push(*matches_per_round.all)
+      end
+
+      MatchesBuilder.new(matches)
+    end
+
+    def to_h
+      @rounds.map(&:to_h)
+    end
+
+    private
+
+    def scrap_rounds(rounds_elements)
+      (0..37).each do |round_number|
+        round_element = rounds_elements.css(
+          "div[data-slide-index=#{round_number}]"
+        )
+
+        round_element.children.each do |element|
+          next unless element.element? && element.name == 'div'
+
+          round = scrap_round(element, round_number)
+
+          @rounds << round
+        end
+      end
+    end
+
+    def scrap_round(round_element, round_number)
+      round = Round.new
+      round.championship = @championship.year
+      round.serie = @championship.serie
+
+      # Because index starts on zero, we add 1 for matching with Rounds ID
+      round.number = round_number + 1
+      scrap_matches(round, round_element)
+
+      round
+    end
+
+    def scrap_matches(round, round_element)
+      round_element.children.each do |element|
+        # matches are founded on <ul>
+        next unless element.element? && element.name == 'ul'
+
+        round.matches = MatchesPerRoundBuilder.new(
+          element,
+          round.number,
+          @championship
+        )
+      end
+      round
+    end
+  end
+end

data/lib/scrap_cbf/builders/teams_builder.rb

@@ -0,0 +1,78 @@
+# frozen_string_literal: true
+
+class ScrapCbf
+  class TeamsBuilder
+    extend Forwardable
+    include TeamsHelper
+    include Formattable
+    include Printable
+
+    delegate [:each] => :@teams
+
+    def initialize(document)
+      @teams = []
+
+      tables = document.css('table')
+      table = find_table_by_header(
+        tables,
+        Ranking::TABLE_HEADER
+      )
+
+      scrap_teams(table)
+    end
+
+    def to_h
+      @teams.map(&:to_h)
+    end
+
+    private
+
+    def scrap_teams(table)
+      table.css('tbody > tr').each do |tr_element|
+        # Remove the rows that are invisible by default
+        next if tr_element.element? && element_hidden?(tr_element)
+
+        teams_elements = tr_element.css('img')
+
+        # two teams are found in a row: team and next opponent
+        # the last one may be not present
+        unless teams_elements.length >= 1 && teams_elements.length <= 2
+          raise InvalidNumberOfEntitiesError.new(:team, teams_elements.length)
+        end
+
+        # only the first team is scraped
+        team_element = teams_elements.first
+
+        @teams << scrap_team(team_element)
+      end
+    end
+
+    def scrap_team(team_element)
+      team = Team.new
+
+      if team_element&.key?('title') &&
+         team_element['title'].match?(/^[a-záàâãéèêíïóôõöúç\s]+ - [a-z]{2}$/i)
+
+        scrap_name(team, team_element)
+        scrap_state(team, team_element)
+        scrap_avatar_url(team, team_element)
+      end
+
+      team
+    end
+
+    def scrap_name(team, team_element)
+      # e.g "Santos"
+      team.name = team_element['title'][/^[a-záàâãéèêíïóôõöúç\s]{3,50}/i].strip
+    end
+
+    def scrap_state(team, team_element)
+      # e.g "SP"
+      team.state = team_element['title'][/[a-z]{2}$/i]
+    end
+
+    def scrap_avatar_url(team, team_element)
+      team.avatar_url = team_element['src'] if team_element.key?('src')
+    end
+  end
+end