boilerpipe-ruby 0.0.1 → 0.1.0

data/lib/boilerpipe/filters/document_title_match_classifier.rb

@@ -0,0 +1,121 @@
+# encoding: utf-8
+require 'set'
+
+ # Marks TextBlocks which contain parts of the HTML <TITLE> tag, using
+ # some heuristics which are quite specific to the news domain.
+
+    # we create a list of potential titles from the page title
+    # then we look at every text block and if the text block
+    # contains a potential title - we set that text block label as :TITLE
+
+module Boilerpipe::Filters
+  class DocumentTitleMatchClassifier
+    attr_reader :potential_titles
+
+    def initialize(title)
+      @potential_titles = Set.new
+      generate_potential_titles(title)
+    end
+
+    def process(doc)
+      return doc if @potential_titles.empty?
+
+      doc.text_blocks.each do |tb|
+        text = tb.text.gsub('\u00a0', ' ')
+          .gsub("'", '')
+          .strip.downcase
+
+        if @potential_titles.member? text
+          tb.add_label :TITLE
+          break
+        end
+
+        remove_characters = /[?!.-:]+/
+        text = text.gsub(remove_characters, '').strip
+
+        if @potential_titles.member? text
+          tb.add_label :TITLE
+          break
+        end
+      end
+
+      doc
+    end
+
+    private
+
+    def generate_potential_titles(title)
+      return if title.nil?
+
+      title = title.gsub('\u00a0', ' ')
+        .gsub("'", '')
+        .strip
+        .downcase
+
+      @potential_titles << title
+
+      # unnecessary
+      #p = longest_part(title, /[ ]*[|»-][ ]*/)
+      #@potential_titles << p if p
+
+      #p = longest_part(title, /[ ]*[|»:][ ]*/)
+      #@potential_titles << p if p
+
+      #p = longest_part(title, /[ ]*[|»:()][ ]*/)
+      #@potential_titles << p if p
+
+      #p = longest_part(title, /[ ]*[|»:()-][ ]*/)
+      #@potential_titles << p if p
+
+      p = longest_part(title, /[ ]*[|»,:()-][ ]*/)
+      @potential_titles << p if p
+
+      # we replace \u00a0 so why check for it?
+      #p = longest_part(title, /[ ]*[|»,:()-\u00a0][ ]*/)
+      #@potential_titles << p if p
+
+      add_potential_titles(title, /[ ]+[|][ ]+/, 4)
+      add_potential_titles(title, /[ ]+[-][ ]+/, 4)
+
+      @potential_titles << title.sub(/ - [^-]+$/, '') # remove right of -
+      @potential_titles << title.sub(/^[^-]+ - /, '') # remove left of -
+    end
+
+    def longest_part(title, regex)
+      parts = title.split regex
+      return nil if parts.size == 1
+
+      longest_num_words = 0
+      longest_part = ''
+
+      parts.each do |part|
+        next if part =~ /[.]com/
+        num_words = number_of_words(part)
+
+        if num_words > longest_num_words || part.size > longest_part.size
+          longest_num_words = num_words
+          longest_part = part
+        end
+      end
+
+      longest_part.empty? ? nil : longest_part.strip
+    end
+
+    def add_potential_titles(title, regex, min_words)
+      parts = title.split regex
+      return if parts.size == 1
+
+      parts.each do |part|
+        next if part =~ /[.]com/
+        num_words = number_of_words(part)
+
+        @potential_titles << part if num_words >= min_words
+      end
+    end
+
+    def number_of_words(s)
+      s.split(/[\b ]+/).size
+    end
+
+  end
+end

data/lib/boilerpipe/filters/expand_title_to_content_filter.rb

@@ -0,0 +1,43 @@
+
+# Marks all TextBlocks "content" which are between the headline and the part that has
+# already been marked content, if they are marked MIGHT_BE_CONTENT.
+# This filter is quite specific to the news domain.
+# used downstream of KeepLargetBlockFilter since that's what sets MIGHT_BE_CONTENT
+
+
+module Boilerpipe::Filters
+  class ExpandTitleToContentFilter
+    def self.process(doc)
+      tbs = doc.text_blocks
+
+      #     slower and more ruby-like
+      #     comeback and let's do some benchmarking
+      #     titles = tbs.select{ |tb| tb.has_label?(:TITLE) }
+      #     title = tbs.index(titles.last)
+      #     content_start = tbs.find_index(&:is_content?)
+
+      i = 0
+      title = nil
+      content_start = nil
+
+      tbs.each do |tb|
+        title = i if content_start.nil? && tb.has_label?(:TITLE)
+        content_start = i if content_start.nil? && tb.is_content?
+        i += 1
+      end
+
+      return doc if no_title_with_subsequent_content?(content_start, title)
+
+      tbs.slice(title...content_start).each do |tb|
+        tb.content = true if tb.has_label?(:MIGHT_BE_CONTENT)
+      end
+
+      doc
+    end
+
+    def self.no_title_with_subsequent_content?(content_start, title)
+      title.nil? || content_start.nil? || content_start <= title
+    end
+
+  end
+end

data/lib/boilerpipe/filters/heuristic_filter_base.rb

@@ -0,0 +1,7 @@
+module Boilerpipe::Filters
+  class HeuristicFilterBase
+    def self.num_full_text_words(tb, min_text_density=9.0)
+      tb.text_density >= min_text_density ? tb.num_words : 0
+    end
+  end
+end

data/lib/boilerpipe/filters/ignore_blocks_after_content_filter.rb

@@ -0,0 +1,24 @@
+ # Marks all blocks as "non-content" that occur after blocks that have been
+ # marked INDICATES_END_OF_TEXT. These marks are ignored unless a minimum
+ # number of words in content blocks occur before this mark (default: 60).
+ # This can be used in conjunction with an upstream TerminatingBlocksFinder.
+
+module Boilerpipe::Filters
+  class IgnoreBlocksAfterContentFilter < HeuristicFilterBase
+
+    def self.process(doc, min_num_words=60)
+      found_end_of_text = false
+      num_words = 0
+
+      doc.text_blocks.each do |tb|
+        end_of_text = tb.has_label? :INDICATES_END_OF_TEXT
+        num_words += num_full_text_words(tb) if tb.is_content?
+        found_end_of_text = true if end_of_text && num_words >= min_num_words
+        tb.content = false if found_end_of_text
+      end
+
+      doc
+    end
+
+  end
+end

data/lib/boilerpipe/filters/keep_largest_block_filter.rb

@@ -0,0 +1,62 @@
+
+# Keeps the largest TextBlock only (by the number of words). In case of
+# more than one block with the same number of words, the first block is chosen.
+# All discarded blocks are marked "not content" and flagged as :MIGHT_BE_CONTENT.
+#
+# Note that, by default, only TextBlocks marked as "content" are taken into
+# consideration.
+
+module Boilerpipe::Filters
+  class KeepLargestBlockFilter
+
+    def initialize(expand_to_same_level_text, min_words)
+      @expand_to_same_level_text = expand_to_same_level_text
+      @min_words = min_words
+    end
+
+    INSTANCE = KeepLargestBlockFilter.new(false, 0)
+    INSTANCE_EXPAND_TO_SAME_TAGLEVEL = KeepLargestBlockFilter.new(true, 0)
+    INSTANCE_EXPAND_TO_SAME_TAGLEVEL_MIN_WORDS = KeepLargestBlockFilter.new(true, 150)
+
+    def process(doc)
+      tbs = doc.text_blocks
+      return false if tbs.size < 2
+
+      # find tb with the most words
+      largest_block = tbs.select(&:is_content?).max_by(&:num_words)
+      level = @expand_to_same_level_text ? largest_block.tag_level : -1
+
+      # set labels for text blocks
+      tbs.each do |tb|
+        if tb == largest_block
+          tb.content = true
+          tb.add_label :VERY_LIKELY_CONTENT
+        else
+          tb.content = false
+          tb.add_label :MIGHT_BE_CONTENT
+        end
+      end
+
+      n = tbs.index(largest_block)
+      if @expand_to_same_level_text && n
+        # expand blocks to the left
+        expand_tag_level(tbs[0...n].reverse, level, @min_words)
+
+        # expand blocks to the right
+        expand_tag_level(tbs[n+1..-1], level, @min_words)
+      end
+    end
+
+    # sets content to true
+    def expand_tag_level(tbs, level, min_words)
+      tbs.each do |tb|
+        if tb.tag_level < level
+          break
+        elsif tb.tag_level == level
+          tb.content = true if tb.num_words >= min_words
+        end
+      end
+    end
+
+  end
+end

data/lib/boilerpipe/filters/large_block_same_tag_level_to_content_filter.rb

@@ -0,0 +1,29 @@
+
+#  Marks all blocks as content that:
+#  are on the same tag-level as very likely main content
+#  (usually the level of the largest  block)
+#  have a significant number of words, currently: at least 100
+#  Used downstream of KeepLargestBlockFilter
+
+module Boilerpipe::Filters
+  class LargeBlockSameTagLevelToContentFilter
+
+    def self.process(doc)
+
+      largest = doc.text_blocks.find do |tb|
+        tb.is_content? && tb.has_label?(:VERY_LIKELY_CONTENT)
+      end
+
+      return doc if largest.nil?
+      tag_level = largest.tag_level
+
+      doc.text_blocks.each do |tb|
+        next if tb.is_content?
+        tb.content = true if tb.num_words >= 100 && tb.tag_level == tag_level
+      end
+
+      doc
+    end
+
+  end
+end

data/lib/boilerpipe/filters/list_at_end_filter.rb

@@ -0,0 +1,25 @@
+# Marks nested list-item blocks after the end of the main content as content.
+#  Used downstream of keep_largest_block_filter.
+
+module Boilerpipe::Filters
+  class ListAtEndFilter
+    MAX = 99999999
+
+    def self.process(doc)
+      tag_level = MAX
+
+      doc.text_blocks.each do |tb|
+        if tb.is_content? && tb.has_label?(:VERY_LIKELY_CONTENT)
+          tag_level = tb.tag_level
+        elsif (tb.tag_level > tag_level && tb.has_label?(:MIGHT_BE_CONTENT) && tb.has_label?(:LI) && tb.link_density == 0)
+          tb.content = true
+        else
+          tag_level = MAX
+        end
+      end
+
+      doc
+    end
+
+  end
+end

data/lib/boilerpipe/filters/num_words_rules_classifier.rb

@@ -0,0 +1,42 @@
+# encoding: utf-8
+
+#  Classifies TextBlocks as content/not-content through rules that have been determined
+#  using the C4.8 machine learning algorithm, as described in the paper
+#  "Boilerplate Detection using Shallow Text Features" (WSDM 2010), particularly
+#  using number of words per block and link density per block.
+
+module Boilerpipe::Filters
+  class NumWordsRulesClassifier
+
+    def self.process(doc)
+      empty = Boilerpipe::Document::TextBlock.empty_start
+      text_blocks = [empty] + doc.text_blocks + [empty]
+
+      text_blocks.each_cons(3) do |slice|
+        prev, current, nxt = *slice
+        current.content = classify(prev, current, nxt)
+      end
+
+      doc
+    end
+
+    private
+
+    def self.classify(prev, current, nxt)
+      return false if current.link_density > 0.333333
+
+      if prev.link_density <= 0.555556
+        return true if current.num_words > 16
+
+        return true if nxt.num_words > 15
+        return true if prev.num_words > 4
+      else
+        return true if current.num_words > 40
+        return true if nxt.num_words > 17
+      end
+
+      false
+    end
+
+  end
+end

data/lib/boilerpipe/filters/terminating_blocks_finder.rb

@@ -0,0 +1,44 @@
+# encoding: utf-8
+
+# Finds blocks which are potentially indicating the end of an article
+# text and marks them with INDICATES_END_OF_TEXT. This can be used
+# in conjunction with a downstream IgnoreBlocksAfterContentFilter.
+
+
+module Boilerpipe::Filters
+  class TerminatingBlocksFinder
+    def self.process(doc)
+      doc.text_blocks.each do |tb|
+        next unless tb.num_words < 15
+        if tb.text.length >= 8 && finds_match?(tb.text.downcase)
+          tb.labels << :INDICATES_END_OF_TEXT
+        elsif tb.link_density == 1.0 && tb.text == 'comment'
+          tb.labels << :INDICATES_END_OF_TEXT
+        end
+      end
+
+      doc
+    end
+
+    def self.finds_match?(text)
+      text.start_with?('comments') ||
+        text =~ /^\d+ (comments|users responded in)/ || # starts with number
+        text.start_with?('© reuters') ||
+        text.start_with?('please rate this') ||
+        text.start_with?('post a comment') ||
+        text.include?('what you think...') ||
+        text.include?('add your comment') ||
+        text.include?('add comment') ||
+        #TODO add this and test
+        #text.include?('leave a reply') ||
+        #text.include?('leave a comment') ||
+        #text.include?('show comments') ||
+        #text.include?('Share this:') ||
+        text.include?('reader views') ||
+        text.include?('have your say') ||
+        text.include?('reader comments') ||
+        text.include?('rätta artikeln') ||
+        text == 'thanks for your comments - this feedback is now closed'
+    end
+  end
+end

data/lib/boilerpipe/filters/trailing_headline_to_boilerplate_filter.rb

@@ -0,0 +1,24 @@
+
+# Marks trailing headlines TextBlocks that have the label :#HEADING
+# as boilerplate. Trailing means they are marked content and are
+# below any other content block.
+
+module Boilerpipe::Filters
+  class TrailingHeadlineToBoilerplateFilter
+    def self.process(doc)
+
+      doc.text_blocks.each do |tb|
+        next unless tb.is_content?
+
+        if tb.has_label? :HEADING
+          tb.content = false
+        else
+          break
+        end
+      end
+
+      doc
+    end
+
+  end
+end

data/lib/boilerpipe/labels/default.rb

@@ -0,0 +1,17 @@
+module Boilerpipe::Labels
+  module Default
+    :TITLE
+    :ARTICLE_METADATA
+    :INDICATES_END_OF_TEXT
+    :MIGHT_BE_CONTENT
+    :VERY_LIKELY_CONTENT
+    :STRICTLY_NOT_CONTENT
+    :HR
+    :LI
+    :HEADING
+    :H1
+    :H2
+    :H3
+    MARKUP_PREFIX = '<'
+  end
+end