boilerpipe-ruby 0.4.0 → 0.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: 68c2ea4ee42a6e1d76e85f7eaa3de9ca95f3a8d3
-  data.tar.gz: 42199704467cb7f20a8fff7c616be67bae1966e1
+SHA256:
+  metadata.gz: 7fec2bd11d29c4b5d14f70e10fcac76beb95c61e25fbe5cac15b82e8c64fbf69
+  data.tar.gz: 766ea373235462c3678cc2487c647d6211fd2fc066626d5c10ab7e4d31f303ad
 SHA512:
-  metadata.gz: 875da6a2ddfdf517509e3ebb7b9c804fadf4a372b812df42bce92fccb82eb8ca31283f11e764c6796c15997bcb8fddf24301d821f4e24a962f40ba6f973f6f17
-  data.tar.gz: 5dfec7323587057c64b931725df2aa5b53dc5a2fadaef52374619fe5798826c98c4e7ae8bb1ea9267b5db9f40249d5822752700e57a145b058231088249d1ac6
+  metadata.gz: be90614a1c2efa29356e9b3b255a5e5d4374474fd6b711d4ed9ab575c4ab8466a1d6903c23de46276133d1621727dea8525422e49608185c1e6294af4f6e0f54
+  data.tar.gz: 5b368e59ced5b794b8e2033b632de67a09bf43e3c45070e96a66ce695bddb5966130e37d9862179a9c989d5166645e182bd2973c179cb8f369c7c4942e238f30

data/.circleci/config.yml

@@ -6,48 +6,30 @@ version: 2
 jobs:
   build:
     docker:
-      # specify the version you desire here
-       - image: circleci/ruby:2.4.1-node-browsers
-      
+       - image: circleci/ruby:2.5.5-node-browsers
+
       # Specify service dependencies here if necessary
       # CircleCI maintains a library of pre-built images
       # documented at https://circleci.com/docs/2.0/circleci-images/
-      # - image: circleci/postgres:9.4
 
     working_directory: ~/repo
 
     steps:
       - checkout
 
-      # Download and cache dependencies
-      - restore_cache:
-          keys:
-          - v1-dependencies-{{ checksum "Gemfile.lock" }}
-          # fallback to using the latest cache if no exact match is found
-          - v1-dependencies-
-
+      - run: gem install bundler
       - run:
           name: install dependencies
           command: |
-            bundle install --jobs=4 --retry=3 --path vendor/bundle
+            bundle install --jobs=4 --retry=3
 
-      - save_cache:
-          paths:
-            - ./vendor/bundle
-          key: v1-dependencies-{{ checksum "Gemfile.lock" }}
-        
-      # Database setup
-      #- run: bundle exec rake db:create
-      #- run: bundle exec rake db:schema:load
-
-      # run tests!
       - run:
           name: run tests
           command: |
             mkdir /tmp/test-results
             TEST_FILES="$(circleci tests glob "spec/**/*_spec.rb")"
-            
-            rspec --format progress "spec"
+
+            bundle exec rspec --format progress "spec"
 
       # collect reports
       - store_test_results:

data/.dockerignore

@@ -0,0 +1,7 @@
+.git
+.gitignore
+ log/*
+ tmp/*
+ *.swp
+ *.swo
+ Gemfile.lock

data/CHANGELOG.md

@@ -1,3 +1,30 @@
+# 0.5.0 / 2021-02-15
+* internal refactoring for clarity
+
+# 0.4.4 / 2021-02-13
+* Do a better job of stripping out script tags
+
+# 0.4.3 / 2020-07-18
+
+* update deps
+
+# 0.4.2 / 2020-03-11
+
+* update deps
+
+# 0.4.1 / 2019-07-04
+
+* Fix bug in min_clause_words_filter ( used in article_sentence_extractor )
+* Allow tests to run in Docker
+* Update circle to continue to work
+* Add architecture flow
+* Code formatting
+* Add min words filter specs
+* Add label action specs
+* Add missing test case to ignorable element spec
+* Add merge_next case to text block spec
+* Dry up includes
+
 # 0.4.0 / 2017-09-15
 
 * Add KeepEverythingWithMinKWords Extractor
@@ -22,4 +49,4 @@
 
 # 0.1.0 / 2017-09-08
 
-* Add Article Extractor 
+* Add Article Extractor

data/Dockerfile

@@ -0,0 +1,14 @@
+From ruby:2.5
+RUN gem install bundler
+COPY *gemspec /usr/src/app/
+COPY Gemfile /usr/src/app/
+COPY lib/boilerpipe/version.rb /usr/src/app/lib/boilerpipe/
+COPY bin /usr/src/app/
+COPY bin/* /usr/src/app/bin/
+
+WORKDIR /usr/src/app
+RUN bin/setup
+
+COPY . /usr/src/app/
+
+CMD ["bundle", "exec", "rspec", "--color", "--format", "doc"]

data/README.md

@@ -1,5 +1,8 @@
 # Boilerpipe
 
+[![CircleCI](https://circleci.com/gh/gregors/boilerpipe-ruby/tree/main.svg?style=shield)](https://circleci.com/gh/gregors/boilerpipe-ruby/tree/main)
+[![Gem Version](https://badge.fury.io/rb/boilerpipe-ruby.svg)](https://badge.fury.io/rb/boilerpipe-ruby)
+
 A pure ruby implemenation of the boilerpipe algorithm.
 
 This is a text extraction utility first written by Christian Kohlshutter - [presentation](http://videolectures.net/wsdm2010_kohlschutter_bdu/)
@@ -10,6 +13,8 @@ I saw other gems making use of boilerpipe via the [free api](http://boilerpipe-w
 
 This solution works great if you're using Jruby but I wanted a pure ruby solution to use on MRI. Open vim - start coding...
 
+Here's a high level [diagram](boilerpipe_flow.md) of how the system works.
+
 # TLDR
 
 Just use either ArticleExtractor, DefaultExtractor or KeepEverythingExtractor - try out the others when you feel like experimenting...
@@ -24,9 +29,6 @@ Presently the follow Extractors are implemented
 * [x] LargestContentExtractor
 * [x] NumWordsRulesExtractor
 
-[![CircleCI](https://circleci.com/gh/gregors/boilerpipe-ruby/tree/master.svg?style=shield)](https://circleci.com/gh/gregors/boilerpipe-ruby/tree/master)
-
-[![Gem Version](https://badge.fury.io/rb/boilerpipe-ruby.svg)](https://badge.fury.io/rb/boilerpipe-ruby)
 
 ## Installation
 
@@ -69,7 +71,14 @@ Or install it yourself as:
 
 After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
 
-To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
+To install this gem onto your local machine, run `bundle exec rake install`.
+
+### Running Tests on Docker
+
+The default run command will run the tests
+
+    docker build -t boilerpipe .
+    docker run -it --rm boilerpipe
 
 ## Contributing
 

data/Rakefile

@@ -1,14 +1,13 @@
-require "bundler/gem_tasks"
-require "rspec/core/rake_task"
+require 'bundler/gem_tasks'
+require 'rspec/core/rake_task'
 
 RSpec::Core::RakeTask.new(:spec)
 
 task :default => :spec
 
-
 desc 'Downloads forked boilerpipe jar from Gregors github for sanity checks'
 task :download_boilerpipe_jar do
   FileUtils.mkdir_p 'spec/sanity_checks/jars/'
   Dir.chdir 'spec/sanity_checks/jars/'
- `wget 'https://github.com/gregors/jruby-boilerpipe/raw/master/lib/boilerpipe-common-2.0-SNAPSHOT-jar-with-dependencies.jar'`
+  `wget 'https://github.com/gregors/jruby-boilerpipe/raw/master/lib/boilerpipe-common-2.0-SNAPSHOT-jar-with-dependencies.jar'`
 end

data/bin/console

@@ -1,7 +1,7 @@
 #!/usr/bin/env ruby
 
-require "bundler/setup"
-require "boilerpipe"
+require 'bundler/setup'
+require 'boilerpipe'
 
 # You can add fixtures and/or initialization code here to make experimenting
 # with your gem easier. You can also use a different console, if you like.
@@ -10,5 +10,5 @@ require "boilerpipe"
 # require "pry"
 # Pry.start
 
-require "irb"
+require 'irb'
 IRB.start

data/boilerpipe-ruby.gemspec

@@ -10,18 +10,18 @@ Gem::Specification.new do |spec|
   spec.email         = ['<gregory.ostermayr@gmail.com>']
   spec.license       = 'Apache 2.0'
 
-  spec.summary       = %q{A pure ruby implemenation of the boilerpipe algorithm}
-  spec.description   = %q{A pure ruby implementation of the boilerpipe algorithm}
+  spec.summary       = %q{A pure ruby implementation of the boilerpipe web content extraction algorithm}
+  spec.description   = %q{A pure ruby implementation of the boilerpipe web content extraction algorithm}
   spec.homepage      = 'https://github.com/gregors/boilerpipe-ruby'
 
   spec.files         = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
-  spec.bindir        = "exe"
+  spec.bindir        = 'exe'
   spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
-  spec.require_paths = ["lib"]
+  spec.require_paths = ['lib']
 
-  spec.add_development_dependency 'bundler', '~> 1.11'
-  spec.add_development_dependency 'rake', '~> 10.0'
-  spec.add_development_dependency 'rspec', '~> 3.0'
-  spec.add_development_dependency 'rickshaw', '~> 0.4.0'
-  spec.add_runtime_dependency 'nokogiri', '>= 1.6.6.2'
+  spec.add_development_dependency 'bundler', '~> 2.0'
+  spec.add_development_dependency 'rake', '>= 12.3.3'
+  spec.add_development_dependency 'rickshaw', '~> 0.5.0'
+  spec.add_development_dependency 'rspec', '~> 3.10'
+  spec.add_runtime_dependency 'nokogiri', '~> 1.10'
 end

data/boilerpipe_flow.md

@@ -0,0 +1,40 @@
+```
+raw html
+   |
+   |
+  sax input -> sax parser(html parser) ->  HTML Content handler -> tokenizer ---------
+                                                                                     |
+    -------------------------------------<------------------------------------<------|
+    |              |            |
+text blocks    text blocks  text blocks
+    |              |            |
+    |              |            |
+    -----------------------------
+          |
+          |
+     text document
+          |
+          |
+        filter
+          |
+        filter
+          |
+        filter
+          |
+        filter
+          |
+        filter
+          |
+        filter
+          |
+        filter
+          |
+        filter
+          |
+        filter
+          |
+          |
+     text document
+          |
+  outputs extracted text
+  ```

data/lib/boilerpipe.rb

@@ -1,3 +1,6 @@
+require 'nokogiri'
+require 'set'
+
 require 'boilerpipe/version'
 
 require 'boilerpipe/util/unicode_tokenizer'
@@ -37,6 +40,7 @@ require 'boilerpipe/filters/trailing_headline_to_boilerplate_filter'
 require 'boilerpipe/labels/default'
 require 'boilerpipe/labels/label_action'
 
+require 'boilerpipe/sax/preprocessor'
 require 'boilerpipe/sax/html_content_handler'
 require 'boilerpipe/sax/boilerpipe_html_parser'
 require 'boilerpipe/sax/tag_action_map'

data/lib/boilerpipe/document/text_block.rb

@@ -1,10 +1,7 @@
-require 'set'
-
 module Boilerpipe
   module Document
     class TextBlock
-
-       #EMPTY_END = TextBlock.new('', 0, 0, 0, 0, 999999999999999999999999999)
+      # EMPTY_END = TextBlock.new('', 0, 0, 0, 0, 999999999999999999999999999)
 
       attr_reader :text, :num_words, :num_words_in_wrapped_lines, :num_words_in_anchor_text,
                   :num_wrapped_lines, :offset_blocks_start, :offset_blocks_end, :text_density,
@@ -12,7 +9,7 @@ module Boilerpipe
 
       attr_accessor :content
 
-      def initialize(text, num_words=0, num_words_in_anchor_text=0, num_words_in_wrapped_lines=0, num_wrapped_lines=1, offset_blocks=0)
+      def initialize(text, num_words = 0, num_words_in_anchor_text = 0, num_words_in_wrapped_lines = 0, num_wrapped_lines = 1, offset_blocks = 0)
         @labels = Set.new
         @text = text
         @num_words = num_words
@@ -32,9 +29,9 @@ module Boilerpipe
         new('', 0, 0, 0, 0, -1)
       end
 
-     def set_tag_level(level)
-       @tag_level = level
-     end
+      def set_tag_level(level)
+        @tag_level = level
+      end
 
       def is_content?
         @content
@@ -68,8 +65,8 @@ module Boilerpipe
         @num_words_in_anchor_text += other.num_words_in_anchor_text
         @num_words_in_wrapped_lines += other.num_words_in_wrapped_lines
         @num_wrapped_lines += other.num_wrapped_lines
-        @offset_blocks_start = [@offset_blocks_start , other.offset_blocks_start].min
-        @offset_blocks_end = [@offset_blocks_end , other.offset_blocks_end].max
+        @offset_blocks_start = [@offset_blocks_start, other.offset_blocks_start].min
+        @offset_blocks_end = [@offset_blocks_end, other.offset_blocks_end].max
         init_densities
         @content |= other.is_content?
 
@@ -87,10 +84,10 @@ module Boilerpipe
       end
 
       def to_s
-        #"[" + offsetBlocksStart + "-" + offsetBlocksEnd + ";tl=" + tagLevel + "; nw=" + numWords + ";nwl=" + numWrappedLines + ";ld=" + linkDensity + "]\t" + (isContent ? "CONTENT" : "boilerplate") + "," + labels + "\n" + getText();
+        # "[" + offsetBlocksStart + "-" + offsetBlocksEnd + ";tl=" + tagLevel + "; nw=" + numWords + ";nwl=" + numWrappedLines + ";ld=" + linkDensity + "]\t" + (isContent ? "CONTENT" : "boilerplate") + "," + labels + "\n" + getText();
         labels = 'null'
         if !@labels.empty?
-          labels ="[#{ @labels.to_a.join(',')}]"
+          labels = "[#{@labels.to_a.join(',')}]"
         end
         "[#{@offset_blocks_start}-#{@offset_blocks_end};tl=#{@tag_level}; nw=#{@num_words};nwl=#{@num_wrapped_lines};ld=#{@link_density}]\t#{is_content? ? 'CONTENT' : 'BOILERPLATE'},#{labels}\n#{text}"
       end
@@ -100,6 +97,7 @@ module Boilerpipe
       end
 
       private
+
       def init_densities
         if @num_words_in_wrapped_lines == 0
           @num_words_in_wrapped_lines = @num_words

data/lib/boilerpipe/document/text_document.rb

@@ -19,14 +19,14 @@ module Boilerpipe
           case text_block.is_content?
           when true
             next unless include_content
-            s << text_block.text
-            s << "\n"
           when false
             next unless include_noncontent
-           s << text_block.text
-           s << "\n"
           end
+
+          s << text_block.text
+          s << "\n"
         end
+
         s
       end
 
@@ -38,7 +38,6 @@ module Boilerpipe
         @text_blocks.map(&:to_s).join("\n")
       end
       alias_method :debug_string, :debug_s
-
     end
   end
 end

data/lib/boilerpipe/extractors/canola_extractor.rb

@@ -1,6 +1,5 @@
 module Boilerpipe::Extractors
   class CanolaExtractor
-
     def self.text(contents)
       doc = ::Boilerpipe::SAX::BoilerpipeHTMLParser.parse(contents)
       ::Boilerpipe::Extractors::CanolaExtractor.process doc

data/lib/boilerpipe/extractors/default_extractor.rb

@@ -1,6 +1,5 @@
 module Boilerpipe::Extractors
   class DefaultExtractor
-
     def self.text(contents)
       doc = ::Boilerpipe::SAX::BoilerpipeHTMLParser.parse(contents)
       ::Boilerpipe::Extractors::DefaultExtractor.process doc

data/lib/boilerpipe/extractors/keep_everything_extractor.rb

@@ -1,4 +1,4 @@
- # Marks all blocks as content.
+# Marks all blocks as content.
 
 module Boilerpipe::Extractors
   class KeepEverythingExtractor

data/lib/boilerpipe/extractors/keep_everything_with_k_min_words_extractor.rb

@@ -1,4 +1,3 @@
-
 # A full-text extractor which extracts the largest text component of a page.
 # For news articles, it may perform better than the DefaultExtractor, but
 # usually worse than ArticleExtractor.

data/lib/boilerpipe/extractors/num_words_rules_extractor.rb

@@ -1,6 +1,5 @@
 module Boilerpipe::Extractors
   class NumWordsRulesExtractor
-
     def self.text(contents)
       doc = ::Boilerpipe::SAX::BoilerpipeHTMLParser.parse(contents)
       ::Boilerpipe::Extractors::NumWordsRulesExtractor.process doc

data/lib/boilerpipe/filters/block_proximity_fusion.rb

@@ -1,11 +1,8 @@
-
-    # Fuses adjacent blocks if their distance (in blocks) does not exceed a certain limit. This
-    # probably makes sense only in cases where an upstream filter already has removed some blocks.
+# Fuses adjacent blocks if their distance (in blocks) does not exceed a certain limit. This
+# probably makes sense only in cases where an upstream filter already has removed some blocks.
 
 module Boilerpipe::Filters
   class BlockProximityFusion
-
-
     def initialize(max_blocks_distance, content_only, same_tag_level_only)
       @max_blocks_distance = max_blocks_distance
       @content_only = content_only
@@ -13,8 +10,8 @@ module Boilerpipe::Filters
     end
 
     MAX_DISTANCE_1 = BlockProximityFusion.new(1, false, false)
-    MAX_DISTANCE_1_SAME_TAGLEVEL = BlockProximityFusion.new( 1, false, true)
-    MAX_DISTANCE_1_CONTENT_ONLY = BlockProximityFusion.new( 1, true, false)
+    MAX_DISTANCE_1_SAME_TAGLEVEL = BlockProximityFusion.new(1, false, true)
+    MAX_DISTANCE_1_CONTENT_ONLY = BlockProximityFusion.new(1, true, false)
     MAX_DISTANCE_1_CONTENT_ONLY_SAME_TAGLEVEL = BlockProximityFusion.new(1, true, true)
 
     def process(doc)
@@ -22,7 +19,7 @@ module Boilerpipe::Filters
       return false if text_blocks.size < 2
 
       prev_block = if @content_only
-                     text_blocks.find{ |tb| tb.is_content? }
+                     text_blocks.find { |tb| tb.is_content? }
                    else
                      text_blocks.first
                    end
@@ -46,18 +43,16 @@ module Boilerpipe::Filters
           ok = false if (prev_block.is_not_content? || tb.is_not_content?) && @content_only
           ok = false if ok && prev_block.tag_level != tb.tag_level && @same_tag_level_only
 
-          if  ok
+          if ok
             prev_block.merge_next(tb)
             blocks_to_remove << tb
           else
             prev_block = tb
           end
         end
-
       end
-      doc.replace_text_blocks!( text_blocks - blocks_to_remove )
+      doc.replace_text_blocks!(text_blocks - blocks_to_remove)
       doc
     end
-
   end
 end