boilerpipe-ruby 0.4.0 → 0.5.0

data/lib/boilerpipe/filters/trailing_headline_to_boilerplate_filter.rb

@@ -1,4 +1,3 @@
-
 # Marks trailing headlines TextBlocks that have the label :#HEADING
 # as boilerplate. Trailing means they are marked content and are
 # below any other content block.
@@ -6,7 +5,6 @@
 module Boilerpipe::Filters
   class TrailingHeadlineToBoilerplateFilter
     def self.process(doc)
-
       doc.text_blocks.each do |tb|
         next unless tb.is_content?
 
@@ -19,6 +17,5 @@ module Boilerpipe::Filters
 
       doc
     end
-
   end
 end

data/lib/boilerpipe/labels/label_action.rb

@@ -2,7 +2,7 @@ module Boilerpipe::Labels
   class LabelAction
     attr_reader :labels
 
-    def initialize(labels=[])
+    def initialize(labels = [])
       @labels = labels
     end
 

data/lib/boilerpipe/sax/boilerpipe_html_parser.rb

@@ -1,20 +1,11 @@
-require 'nokogiri'
 module Boilerpipe::SAX
   class BoilerpipeHTMLParser
     def self.parse(text)
-
-      #script bug - delete script tags
-      text.gsub!(/\<script>.+?<\/script>/i, '')
-
-      # nokogiri uses libxml for mri and nekohtml for jruby
-      # mri doesn't remove &nbsp; when missing the semicolon
-      text.gsub!(/(&nbsp) /, '\1; ')
-
+      # strip out tags that cause issues
+      text = Preprocessor.strip(text)
 
       # use nokogiri to fix any bad tags, errors - keep experimenting with this
       text = Nokogiri::HTML(text).to_html
-
-
       handler = HTMLContentHandler.new
       noko_parser = Nokogiri::HTML::SAX::Parser.new(handler)
       noko_parser.parse(text)

data/lib/boilerpipe/sax/html_content_handler.rb

@@ -1,11 +1,8 @@
-require 'nokogiri'
-require 'set'
-
 module Boilerpipe::SAX
   class HTMLContentHandler < Nokogiri::XML::SAX::Document
     attr_reader :in_ignorable_element, :label_stacks, :last_start_tag
 
-    attr_accessor :in_anchor_tag, :token_buffer ,:font_size_stack
+    attr_accessor :in_anchor_tag, :token_buffer, :font_size_stack
     ANCHOR_TEXT_START = "$\ue00a<"
     ANCHOR_TEXT_END = ">\ue00a$"
 
@@ -20,7 +17,7 @@ module Boilerpipe::SAX
       @flush = false
       @block_tag_level = -1
 
-      @in_body = 0
+      @in_body_tag = 0
       @in_anchor_tag = 0
       @in_ignorable_element = 0
       @in_anchor_text = false
@@ -34,7 +31,6 @@ module Boilerpipe::SAX
       @label_stacks << nil
       tag = name.upcase.intern
 
-
       tag_action = @tag_actions[tag]
       if tag_action
         @tag_level += 1 if tag_action.changes_tag_level?
@@ -51,15 +47,15 @@ module Boilerpipe::SAX
     def characters(text)
       flush_block if @flush
 
-      return if @in_ignorable_element != 0
+      return if in_ignorable_element?
       return if text.empty?
 
       # replace all whitespace with simple space
       text.gsub!(/\s+/, ' ')
 
       # trim whitespace
-      started_with_whitespace = text  =~ /^\s/
-      ended_with_whitespace = text  =~ /\s$/
+      started_with_whitespace = text =~ /^\s/
+      ended_with_whitespace = text =~ /\s$/
       text.strip!
 
       #  add a single space if the block was only whitespace
@@ -96,9 +92,15 @@ module Boilerpipe::SAX
       @label_stacks.pop
     end
 
+    def not_in_body_tag?
+      @in_body_tag == 0
+    end
+
     def flush_block
       @flush = false
-      if @in_body == 0
+
+      # set title
+      if not_in_body_tag?
         @title = @token_buffer.strip if :TITLE == @last_start_tag
         clear_buffers
         return
@@ -158,10 +160,10 @@ module Boilerpipe::SAX
       end
 
       text_block = ::Boilerpipe::Document::TextBlock.new(@text_buffer.strip,
-                                 num_words,
-                                 num_linked_words,
-                                 num_words_in_wrapped_lines,
-                                 num_wrapped_lines, @offset_blocks)
+                                                         num_words,
+                                                         num_linked_words,
+                                                         num_words_in_wrapped_lines,
+                                                         num_wrapped_lines, @offset_blocks)
 
       @offset_blocks += 1
       clear_buffers
@@ -187,10 +189,10 @@ module Boilerpipe::SAX
     # \p{No}  -- a numeric character of other type
 
     def is_word?(word)
-       word =~ VALID_WORD_CHARACTER
+      word =~ VALID_WORD_CHARACTER
     end
 
-    #public void flushBlock() {
+    # public void flushBlock() {
     #    int numWords = 0;
     #    int numLinkedWords = 0;
     #    int numWrappedLines = 0;
@@ -198,22 +200,23 @@ module Boilerpipe::SAX
     #    final int maxLineLength = 80;
     #    int numTokens = 0;
     #    int numWordsCurrentLine = 0;
-    #}
+    # }
 
     def increase_in_ignorable_element!
       @in_ignorable_element += 1
     end
 
+    # should we prevent less than zero here?
     def decrease_in_ignorable_element!
       @in_ignorable_element -= 1
     end
 
-    def increase_in_body!
-      @in_body += 1
+    def enter_body_tag!
+      @in_body_tag += 1
     end
 
-    def decrease_in_body!
-      @in_body -= 1
+    def exit_body_tag!
+      @in_body_tag -= 1
     end
 
     def in_ignorable_element?
@@ -224,7 +227,6 @@ module Boilerpipe::SAX
       @in_anchor_tag > 0
     end
 
-
     def add_text_block(text_block)
       @label_stacks.each do |stack|
         next unless stack
@@ -239,6 +241,7 @@ module Boilerpipe::SAX
     # append space if last character wasn't already one
     def append_space
       return if @sb_last_was_whitespace
+
       @sb_last_was_whitespace = true
 
       @text_buffer << ' '

data/lib/boilerpipe/sax/preprocessor.rb

@@ -0,0 +1,11 @@
+module Boilerpipe::SAX
+  class Preprocessor
+    def self.strip(text)
+      # script bug - delete script tags
+      text = text.gsub(/\<script.+?<\/script>/im, '')
+      # nokogiri uses libxml for mri and nekohtml for jruby
+      # mri doesn't remove &nbsp; when missing the semicolon
+      text.gsub(/(&nbsp) /, '\1; ')
+    end
+  end
+end

data/lib/boilerpipe/sax/tag_action_map.rb

@@ -48,4 +48,3 @@ module Boilerpipe::SAX
     end
   end
 end
-

data/lib/boilerpipe/sax/tag_actions/anchor_text.rb

@@ -2,7 +2,7 @@ module Boilerpipe::SAX::TagActions
   class AnchorText
     # Marks this tag as "anchor" (this should usually only be set for the <A> tag). Anchor tags may not be nested.
     # There is a bug in certain versions of NekoHTML which still allows nested tags. If boilerpipe
-    #* encounters such nestings, a SAXException is thrown.
+    # * encounters such nestings, a SAXException is thrown.
     def start(handler, name, attrs)
       if handler.in_anchor_tag?
         handler.in_anchor_tag += 1
@@ -42,7 +42,7 @@ module Boilerpipe::SAX::TagActions
       # - dunno about nokogiri???????
       # as nested A elements are not allowed per specification, we
       # are probably reaching this branch due to a bug in the XML parser
-      #puts "Warning: SAX input contains nested A elements -- You have probably hit a bug in your HTML parser (e.g., NekoHTML bug #2909310). Please clean the HTML externally and feed it to boilerpipe again. Trying to recover somehow..."
+      # puts "Warning: SAX input contains nested A elements -- You have probably hit a bug in your HTML parser (e.g., NekoHTML bug #2909310). Please clean the HTML externally and feed it to boilerpipe again. Trying to recover somehow..."
       end_tag(handler, name)
     end
   end

data/lib/boilerpipe/sax/tag_actions/block_level.rb

@@ -1,6 +1,6 @@
 module Boilerpipe::SAX::TagActions
-     # Explicitly marks this tag a simple "block-level" element,
-     # which always generates whitespace
+  # Explicitly marks this tag a simple "block-level" element,
+  # which always generates whitespace
   class BlockLevel
     def start(handler, name, attrs)
       true

data/lib/boilerpipe/sax/tag_actions/block_tag_label.rb

@@ -1,6 +1,6 @@
 module Boilerpipe::SAX::TagActions
-# for block-level elements, which triggers some LabelAction on
-# the generated TextBlock.
+  # for block-level elements, which triggers some LabelAction on
+  # the generated TextBlock.
   class BlockTagLabel
     def initialize(label_action)
       @label_action = label_action

data/lib/boilerpipe/sax/tag_actions/body.rb

@@ -1,16 +1,16 @@
 module Boilerpipe::SAX::TagActions
-   # Marks this tag the body element (this should usually only
-   # be set for the <BODY> tag).
+  # Marks this tag the body element (this should usually only
+  # be set for the <BODY> tag).
   class Body
     def start(handler, name, attrs)
       handler.flush_block
-      handler.increase_in_body!
+      handler.enter_body_tag!
       false
     end
 
     def end_tag(handler, name)
       handler.flush_block
-      handler.decrease_in_body!
+      handler.exit_body_tag!
       false
     end
 

data/lib/boilerpipe/sax/tag_actions/font.rb

@@ -10,7 +10,7 @@ module Boilerpipe::SAX::TagActions
         rel = m[1]
         val = m[2].to_i # absolute
         size = rel.empty? ? val : relative(handler.font_size_stack, rel, val)
-        handler.font_size_stack <<  size
+        handler.font_size_stack << size
       else
         handler.font_size_stack << nil
       end
@@ -27,7 +27,7 @@ module Boilerpipe::SAX::TagActions
     end
 
     def relative(font_size_stack, rel, val)
-      prev_size = font_size_stack.reverse_each.find{|s| s != nil}
+      prev_size = font_size_stack.reverse_each.find { |s| !s.nil? }
       prev_size = 3 if prev_size.nil?
 
       size = if rel == '+'

data/lib/boilerpipe/version.rb

@@ -1,3 +1,3 @@
 module Boilerpipe
-  VERSION = '0.4.0'
+  VERSION = '0.5.0'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: boilerpipe-ruby
 version: !ruby/object:Gem::Version
-  version: 0.4.0
+  version: 0.5.0
 platform: ruby
 authors:
 - Gregory Ostermayr
-autorequire: 
+autorequire:
 bindir: exe
 cert_chain: []
-date: 2017-09-15 00:00:00.000000000 Z
+date: 2021-02-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -16,71 +16,71 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.11'
+        version: '2.0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.11'
+        version: '2.0'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '10.0'
+        version: 12.3.3
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '10.0'
+        version: 12.3.3
 - !ruby/object:Gem::Dependency
-  name: rspec
+  name: rickshaw
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.0'
+        version: 0.5.0
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.0'
+        version: 0.5.0
 - !ruby/object:Gem::Dependency
-  name: rickshaw
+  name: rspec
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.4.0
+        version: '3.10'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.4.0
+        version: '3.10'
 - !ruby/object:Gem::Dependency
   name: nokogiri
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.6.6.2
+        version: '1.10'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.6.6.2
-description: A pure ruby implementation of the boilerpipe algorithm
+        version: '1.10'
+description: A pure ruby implementation of the boilerpipe web content extraction algorithm
 email:
 - "<gregory.ostermayr@gmail.com>"
 executables: []
@@ -88,9 +88,11 @@ extensions: []
 extra_rdoc_files: []
 files:
 - ".circleci/config.yml"
+- ".dockerignore"
 - ".gitignore"
 - ".rspec"
 - CHANGELOG.md
+- Dockerfile
 - Gemfile
 - LICENSE.txt
 - README.md
@@ -98,6 +100,7 @@ files:
 - bin/console
 - bin/setup
 - boilerpipe-ruby.gemspec
+- boilerpipe_flow.md
 - lib/boilerpipe.rb
 - lib/boilerpipe/document/text_block.rb
 - lib/boilerpipe/document/text_document.rb
@@ -133,6 +136,7 @@ files:
 - lib/boilerpipe/labels/label_action.rb
 - lib/boilerpipe/sax/boilerpipe_html_parser.rb
 - lib/boilerpipe/sax/html_content_handler.rb
+- lib/boilerpipe/sax/preprocessor.rb
 - lib/boilerpipe/sax/tag_action_map.rb
 - lib/boilerpipe/sax/tag_actions/anchor_text.rb
 - lib/boilerpipe/sax/tag_actions/block_level.rb
@@ -151,7 +155,7 @@ homepage: https://github.com/gregors/boilerpipe-ruby
 licenses:
 - Apache 2.0
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -166,9 +170,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.6.12
-signing_key: 
+rubygems_version: 3.0.8
+signing_key:
 specification_version: 4
-summary: A pure ruby implemenation of the boilerpipe algorithm
+summary: A pure ruby implementation of the boilerpipe web content extraction algorithm
 test_files: []