scrap_cbf 0.1.0

data/lib/scrap_cbf/document.rb

@@ -0,0 +1,93 @@
+# frozen_string_literal: true
+
+class ScrapCbf
+  # This class is responsible for:
+  # - Handler users input.
+  # - Fetch html page from CBF official page.
+  # - Parse html page with Nokogiri.
+  class Document
+    URL = 'https://www.cbf.com.br/futebol-brasileiro/competicoes'
+
+    CHAMPIONSHIP_YEARS = (2012..Date.today.year.to_i).to_a.freeze
+
+    SERIES = %i[serie_a serie_b].freeze
+
+    SERIES_PATH = {
+      serie_a: 'campeonato-brasileiro-serie-a',
+      serie_b: 'campeonato-brasileiro-serie-b'
+    }.freeze
+
+    SAMPLE_PATH = "#{File.dirname(__FILE__)}/samples/" \
+    'cbf_serie_a_2020.html'
+
+    class << self
+      def parse_document(year, serie, opts)
+        new(year, serie, opts).parsed_document
+      end
+    end
+
+    attr_reader :year,
+                :serie,
+                :load_from_sample,
+                :sample_path,
+                :parsed_document
+
+    # @param [Integer] year the Championship year
+    # @param [Symbol] serie the Championship serie. see SERIES.
+    # @option opts [Boolean] load_from_sample yes or no to load specific
+    #  HTML file
+    # @option opts [Symbol] sample_path path to the sample otherwise default
+    #
+    # @return [Document] new instance
+    def initialize(year, serie, opts)
+      @year = year
+      @serie = serie
+      @load_from_sample = opts.fetch(:load_from_sample) { false }
+      @sample_path = opts[:sample_path]
+
+      @parsed_document =
+        parse_document(year, serie, @load_from_sample, @sample_path)
+    end
+
+    private
+
+    # @param [Integer] year the Championship year
+    # @param [Symbol] serie the Championship serie. see SERIES.
+    # @option opts [Boolean] load_from_sample yes or no to load specific
+    #  HTML file
+    # @option opts [Symbol] sample_path path to the sample otherwise default
+    #
+    # @return [Nokogiri::HTML::Document] new instance
+    def parse_document(year, serie, load_from_sample, sample_path)
+      url = if load_from_sample
+              sample_path || SAMPLE_PATH
+            else
+              raise_year_error if year_out_of_range?(year)
+              raise_serie_error if serie_out_of_range?(serie)
+
+              build_url(year, serie)
+            end
+      Nokogiri::HTML(URI.open(url))
+    end
+
+    def build_url(year, serie)
+      "#{URL}/#{SERIES_PATH[serie]}/#{year}"
+    end
+
+    def year_out_of_range?(year)
+      !CHAMPIONSHIP_YEARS.include?(year)
+    end
+
+    def serie_out_of_range?(serie)
+      !SERIES.include?(serie)
+    end
+
+    def raise_year_error
+      raise OutOfRangeArgumentError.new(:year, CHAMPIONSHIP_YEARS)
+    end
+
+    def raise_serie_error
+      raise OutOfRangeArgumentError.new(:serie, SERIES)
+    end
+  end
+end

data/lib/scrap_cbf/errors.rb

@@ -0,0 +1,51 @@
+# frozen_string_literal: true
+
+class ScrapCbf
+  # Base error for all ScrapCbf errors.
+  class BaseError < ::StandardError; end
+
+  # Raised when a required method is not implemented.
+  class MethodNotImplementedError < BaseError; end
+
+  # Raised when a argument is not included in a predefined range of values.
+  class OutOfRangeArgumentError < BaseError
+    def initialize(argument, range)
+      message = "#{argument} must be in the range of : #{range}"
+      super(message)
+    end
+  end
+
+  # Raised when a argument is required but it is missing.
+  class MissingArgumentError < BaseError
+    def initialize(argument)
+      message = "missing argument: #{argument}"
+      super(message)
+    end
+  end
+
+  # Raised when the scraping data is not included in a predefined range.
+  class InvalidNumberOfEntitiesError < BaseError
+    def initialize(entity, number)
+      message = "an invalid number of #{entity} entities was found: #{number}"
+      super(message)
+    end
+  end
+
+  # Raised when the scraping data from a table
+  # have the size of the Table's header different than the Table's rows.
+  class RowSizeError < BaseError
+    def initialize(row_length, header_length)
+      message = "row length: #{row_length} doesn't match with " \
+      "header length: #{header_length}"
+      super(message)
+    end
+  end
+
+  # Raised when a method is not found on a class.
+  class MethodMissingError < BaseError
+    def initialize(klass_name, method)
+      message = "method '#{method}' missing on class #{klass_name}"
+      super(message)
+    end
+  end
+end

data/lib/scrap_cbf/formattable.rb

@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+class ScrapCbf
+  # Format data in standarts outputs
+  module Formattable
+    # @return [Json]
+    def to_json(*_args)
+      to_h.to_json
+    end
+
+    # @return [Hash]
+    def to_h
+      raise MethodNotImplementedError
+    end
+  end
+end

data/lib/scrap_cbf/helpers/lib/depth_search.rb

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+class ScrapCbf
+  # This module use recursion and regex to find specifics methods on the dom.
+  # A better use case for this method is searching on a component view
+  # that has small text wrapped only on a html tag without a specific class
+  module DepthSearch
+    # Search element's children recursively and returns first assertion matched.
+    #
+    # The assertion must be passed through a proc, and must return nil for
+    # false assertions and the element searched for true assetions.
+    #
+    # @param element [Nokogiri::XML::Element] to be searched.
+    # @param proc [Proc] with logic to test assertion.
+    # @return [Object, nil] object may be text element or a dom element.
+    def depth_search(element, proc)
+      res = nil
+      counter = 0
+      number_of_children = element.children.length
+
+      while counter < number_of_children
+
+        child = element.children[counter]
+        res = proc.call(child)
+
+        return res if res # recursion base case #1 - return when found
+
+        res = depth_search(child, proc)
+
+        return res if res # recursion base case #2 - return from recursion
+
+        counter += 1
+      end
+      res
+    end
+  end
+end

data/lib/scrap_cbf/helpers/lib/element_nokogiri.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+class ScrapCbf
+  # This helper is a outliner. The purpose of this module is given a
+  # better interface for the object generate by Nokogiri.
+  # We can think as them as an extansion for the Nokogiri obj element.
+  # This module may disappear later.
+  module ElementNokogiri
+    def element_hidden?(element)
+      element['style']&.eql?('display: none')
+    end
+
+    def remove_whitespace(element)
+      element.text.gsub(/\s+/, '')
+    end
+  end
+end

data/lib/scrap_cbf/helpers/lib/findable.rb

@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+class ScrapCbf
+  # This module has methods for helping on the task of find specific component
+  # views (e.g table).
+  module Findable
+    # Find in the Document the first table with single header and level that
+    # matches with such accurancy the argument compare.
+    #
+    # This method uses Array#find to return object or nil.
+    #
+    # @return [Nokogiri::XML::Element, nil]
+    def find_table_by_header(
+      elems,
+      compare,
+      regex = '[[:alpha:]]+',
+      accuracy = 1.0
+    )
+      elems.find do |table|
+        # check only single level header
+        thead = table.css('thead').first
+        return false if !thead || thead.css('tr').length > 1
+
+        header = thead.text.scan(Regexp.new(regex))
+
+        return false if header.empty?
+
+        matches = (compare & header).length
+        (header.length / matches) >= accuracy
+      end
+    end
+  end
+end

data/lib/scrap_cbf/helpers/matches_helper.rb

@@ -0,0 +1,136 @@
+# frozen_string_literal: true
+
+require_relative 'lib/depth_search'
+
+class ScrapCbf
+  module MatchesHelper
+    include DepthSearch
+
+    # e.g 0 x 1
+    def score_by_team_helper(team, score)
+      case team
+      when :team
+        score.split(' ')[0].to_i if score
+      when :opponent
+        score.split(' ')[2].to_i if score
+      end
+    end
+
+    # 03/02/2021 16:00
+    def date_with_start_at_helper(date, start_at)
+      return unless date && start_at
+
+      "#{date} #{start_at}"
+    end
+
+    # teams are extract from <img>
+    def scrap_team_names_helper(team_element)
+      if team_element.key?('title') &&
+         team_element['title'].match?(/^[a-záàâãéèêíïóôõöúç\s]+ - [a-z]{2}$/i)
+
+        # Extract team's name (e.g Santos - SP => Santos)
+        team_element['title'][/^[a-záàâãéèêíïóôõöúç\s]{3,50}/i].strip
+      end
+    end
+
+    # Because of problem passing regex, couldn't turn the 5 methods in 1.
+    #
+    # pass assertive Proc to depth_search
+    def find_info_helper(match)
+      find = proc do |element|
+        if element.text?
+          formatted_text = element.text.strip
+          unless formatted_text.empty?
+            res = formatted_text.match?(
+              /Jogo: \d{1,3}$/i
+            )
+            next formatted_text if res
+          end
+        end
+        nil
+      end
+
+      depth_search(match, find)
+    end
+
+    # Because of problem passing regex, couldn't turn the 5 methods in 1.
+    #
+    # pass assertive Proc to depth_search
+    def find_updates_helper(match)
+      find = proc do |element|
+        if element.text?
+          formatted_text = element.text.strip
+          unless formatted_text.empty?
+            res = formatted_text.match?(
+              /\d{1} (ALTERAÇÃO|ALTERAÇÕES)$/i
+            )
+            next formatted_text if res
+          end
+        end
+        nil
+      end
+
+      depth_search(match, find)
+    end
+
+    # Because of problem passing regex, couldn't turn the 5 methods in 1.
+    #
+    # pass assertive Proc to depth_search
+    def find_start_at_helper(match)
+      find = proc do |element|
+        if element.text?
+          formatted_text = element.text.strip
+          unless formatted_text.empty?
+            res = formatted_text.match?(
+              /^\d{2}:\d{2}$/i
+            )
+            next formatted_text if res
+          end
+        end
+        nil
+      end
+
+      depth_search(match, find)
+    end
+
+    # Because of problem passing regex, couldn't turn the 5 methods in 1.
+    #
+    # pass assertive Proc to depth_search
+    def find_score_helper(match)
+      find = proc do |element|
+        if element.text?
+          formatted_text = element.text.strip
+          unless formatted_text.empty?
+            res = formatted_text.match?(
+              /^\d{1} x \d{1}$/i
+            )
+            next formatted_text if res
+          end
+        end
+        nil
+      end
+
+      depth_search(match, find)
+    end
+
+    # Because of problem passing regex, couldn't turn the 4 methods in 1.
+    #
+    # pass assertive Proc to depth_search
+    def find_place_helper(match)
+      find = proc do |element|
+        if element.text?
+          formatted_text = element.text.strip
+          unless formatted_text.empty?
+            res = formatted_text.match?(
+              /^[a-záàâãéèêíïóôõöúçñ\-\s]+ - [a-záàâãéèêíïóôõöúçñ\s\-]+ - [A-Z]{2}$/i
+            )
+            next formatted_text if res
+          end
+        end
+        nil
+      end
+
+      depth_search(match, find)
+    end
+  end
+end

data/lib/scrap_cbf/helpers/rankings_helper.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+require_relative 'lib/element_nokogiri'
+require_relative 'lib/findable'
+
+class ScrapCbf
+  module RankingsHelper
+    include ElementNokogiri
+    include Findable
+
+    def title_or_nil_helper(element)
+      child_elem = element.children.find do |elem|
+        elem.element? && elem.key?('title')
+      end
+
+      child_elem['title'] if child_elem
+    end
+  end
+end

data/lib/scrap_cbf/helpers/teams_helper.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+require_relative 'lib/element_nokogiri'
+require_relative 'lib/findable'
+
+class ScrapCbf
+  module TeamsHelper
+    include ElementNokogiri
+    include Findable
+  end
+end

data/lib/scrap_cbf/models/championship.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+class ScrapCbf
+  class Championship
+    include Formattable
+    include Printable
+
+    ATTR_ACCESSORS = %i[
+      year
+      serie
+    ].freeze
+
+    attr_accessor(*ATTR_ACCESSORS)
+
+    def initialize(year, serie)
+      @year = year
+      @serie = serie
+    end
+
+    def to_h
+      ATTR_ACCESSORS.each_with_object({}) do |attribute, hash|
+        hash[attribute] = send attribute
+      end.with_indifferent_access
+    end
+  end
+end