boilerpipe-ruby 0.1.1 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 8b4fe90788ec566674b1a076d7e86c0467d1b805
-  data.tar.gz: 762c1e5e23730ddc0fe4b3d4096c92939b715dc7
+  metadata.gz: cbfe98ebb939cfdc50ff0fb7a6bdc4a78b91e880
+  data.tar.gz: fe8781a571918940a74f191b81316f5bf65aaa7a
 SHA512:
-  metadata.gz: f185331100d3533326800cf2ee610ea2b962d4ff6971b89a0b05f1443bea490f730f78cd17e04b421f2f8fc4009252fac1ed90476350f9b68879f963111d345e
-  data.tar.gz: 08313ed988aecb0a69e0479d3605f3dfb922928a605136a5a23d44d3fd5d4073550f035fee7861aa9b8280874faa55ec6fc63393f9643e7ea11150a2fb2b4261
+  metadata.gz: e8f92169b5b7b766f4fc635ffb5e423ab5493a4bc8df6d546af7373df3863266e7a4fb7ba4973b4b84f79985e1f6585bd98a3bd3547f8bb6c558113288c2549d
+  data.tar.gz: 3f4d9cb77ce06710dd744bc0be6cfe67cfc211631b95e670f5bdccc9ffa6ab48bfe167206f08a20b83adc249ed86afa6c0556e43f83cacc57444a44a0a539e1f

data/.circleci/config.yml

@@ -0,0 +1,57 @@
+# Ruby CircleCI 2.0 configuration file
+#
+# Check https://circleci.com/docs/2.0/language-ruby/ for more details
+#
+version: 2
+jobs:
+  build:
+    docker:
+      # specify the version you desire here
+       - image: circleci/ruby:2.4.1-node-browsers
+      
+      # Specify service dependencies here if necessary
+      # CircleCI maintains a library of pre-built images
+      # documented at https://circleci.com/docs/2.0/circleci-images/
+      # - image: circleci/postgres:9.4
+
+    working_directory: ~/repo
+
+    steps:
+      - checkout
+
+      # Download and cache dependencies
+      - restore_cache:
+          keys:
+          - v1-dependencies-{{ checksum "Gemfile.lock" }}
+          # fallback to using the latest cache if no exact match is found
+          - v1-dependencies-
+
+      - run:
+          name: install dependencies
+          command: |
+            bundle install --jobs=4 --retry=3 --path vendor/bundle
+
+      - save_cache:
+          paths:
+            - ./vendor/bundle
+          key: v1-dependencies-{{ checksum "Gemfile.lock" }}
+        
+      # Database setup
+      #- run: bundle exec rake db:create
+      #- run: bundle exec rake db:schema:load
+
+      # run tests!
+      - run:
+          name: run tests
+          command: |
+            mkdir /tmp/test-results
+            TEST_FILES="$(circleci tests glob "spec/**/*_spec.rb")"
+            
+            rspec --format progress "spec"
+
+      # collect reports
+      - store_test_results:
+          path: /tmp/test-results
+      - store_artifacts:
+          path: /tmp/test-results
+          destination: test-results

data/.gitignore

@@ -1,6 +1,5 @@
 /.bundle/
 /.yardoc
-/Gemfile.lock
 /_yardoc/
 /coverage/
 /doc/

data/CHANGELOG.md

@@ -0,0 +1,13 @@
+# 0.2.0 / 2017-09-11
+
+* Add Default Extractor
+* Tweak dependency to use Nokogiri 1.6.6.2 or newer
+* Add Apache 2.0 license to reflect original work by Christian Kohlschütter
+
+# 0.1.1 / 2017-09-11
+
+* bugfix new line character escaping bug
+
+# 0.1.0 / 2017-09-08
+
+* Add Article Extractor 

data/LICENSE.txt

@@ -0,0 +1,33 @@
+boilerpipe
+
+Copyright (c) 2009, 2014 Christian Kohlschütter
+
+The author licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+
+================================================================================
+
+Reimplemetation of the original Java-based boilerpipe library
+
+Modifications Copyright (c) 2017 Gregory Ostermayr 
+
+ The author licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License.  You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.

data/README.md

@@ -14,10 +14,12 @@ I've only got the ArticleExtractor working but the others should be following qu
 
 Presently the follow Extractors are implemented
 * [x] ArticleExtractor
-* [ ] DefaultExtractor
+* [x] DefaultExtractor
 * [ ] LargestContentExtractor
 * [ ] KeepEverythingExtractor
 
+[![CircleCI](https://circleci.com/gh/gregors/boilerpipe-ruby/tree/master.svg?style=shield)](https://circleci.com/gh/gregors/boilerpipe-ruby/tree/master)
+
 ## Installation
 
 Add this line to your application's Gemfile:
@@ -44,6 +46,8 @@ Or install it yourself as:
     > content = open('https://blog.carbonfive.com/2017/08/28/always-squash-and-rebase-your-git-commits/').read; true;
     > output = Boilerpipe::Extractors::ArticleExtractor.text(content).slice(0..40)
      => "Always Squash and Rebase your Git Commits" 
+    > output = Boilerpipe::Extractors::DefaultExtractor.text(content).slice(0..40)
+     => "Posted on\nWhat is the squash rebase workf"
 
 ## Development
 

data/boilerpipe-ruby.gemspec

@@ -8,6 +8,7 @@ Gem::Specification.new do |spec|
   spec.version       = Boilerpipe::VERSION
   spec.authors       = ['Gregory Ostermayr']
   spec.email         = ['<gregory.ostermayr@gmail.com>']
+  spec.license       = 'Apache 2.0'
 
   spec.summary       = %q{A pure ruby implemenation of the boilerpipe algorithm}
   spec.description   = %q{A pure ruby implementation of the boilerpipe algorithm}
@@ -21,5 +22,6 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency 'bundler', '~> 1.11'
   spec.add_development_dependency 'rake', '~> 10.0'
   spec.add_development_dependency 'rspec', '~> 3.0'
-  spec.add_runtime_dependency 'nokogiri', '1.6.6.2'
+  spec.add_development_dependency 'rickshaw', '~> 0.4.0'
+  spec.add_runtime_dependency 'nokogiri', '>= 1.6.6.2'
 end

data/lib/boilerpipe.rb

@@ -1,10 +1,16 @@
 require 'boilerpipe/version'
+
 require 'boilerpipe/util/unicode_tokenizer'
+
 require 'boilerpipe/document/text_document'
 require 'boilerpipe/document/text_block'
+
 require 'boilerpipe/extractors/article_extractor'
+require 'boilerpipe/extractors/default_extractor'
+
 require 'boilerpipe/filters/block_proximity_fusion'
 require 'boilerpipe/filters/boilerplate_block_filter'
+require 'boilerpipe/filters/density_rules_classifier'
 require 'boilerpipe/filters/document_title_match_classifier'
 require 'boilerpipe/filters/expand_title_to_content_filter'
 require 'boilerpipe/filters/heuristic_filter_base'
@@ -13,10 +19,13 @@ require 'boilerpipe/filters/keep_largest_block_filter'
 require 'boilerpipe/filters/large_block_same_tag_level_to_content_filter'
 require 'boilerpipe/filters/list_at_end_filter'
 require 'boilerpipe/filters/num_words_rules_classifier'
+require 'boilerpipe/filters/simple_block_fusion_processor'
 require 'boilerpipe/filters/terminating_blocks_finder'
 require 'boilerpipe/filters/trailing_headline_to_boilerplate_filter'
+
 require 'boilerpipe/labels/default'
 require 'boilerpipe/labels/label_action'
+
 require 'boilerpipe/sax/html_content_handler'
 require 'boilerpipe/sax/boilerpipe_html_parser'
 require 'boilerpipe/sax/tag_action_map'

data/lib/boilerpipe/document/text_block.rb

@@ -12,7 +12,7 @@ module Boilerpipe
 
       attr_accessor :content
 
-      def initialize(text, num_words=0, num_words_in_anchor_text=0, num_words_in_wrapped_lines=0, num_wrapped_lines=0, offset_blocks=0)
+      def initialize(text, num_words=0, num_words_in_anchor_text=0, num_words_in_wrapped_lines=0, num_wrapped_lines=1, offset_blocks=0)
         @labels = Set.new
         @text = text
         @num_words = num_words

data/lib/boilerpipe/extractors/article_extractor.rb

@@ -3,6 +3,7 @@ module Boilerpipe::Extractors
     def self.text(contents)
       doc = ::Boilerpipe::SAX::BoilerpipeHTMLParser.parse(contents)
       ::Boilerpipe::Extractors::ArticleExtractor.process(doc)
+      doc.content
     end
 
     def self.process(doc)
@@ -46,7 +47,7 @@ module Boilerpipe::Extractors
       # Marks nested list-item blocks after the end of the main content as content.
       filters::ListAtEndFilter.process doc
 
-      doc.content
+      doc
     end
   end
 end

data/lib/boilerpipe/extractors/default_extractor.rb

@@ -0,0 +1,24 @@
+module Boilerpipe::Extractors
+  class DefaultExtractor
+
+    def self.text(contents)
+      doc = ::Boilerpipe::SAX::BoilerpipeHTMLParser.parse(contents)
+      ::Boilerpipe::Extractors::DefaultExtractor.process doc
+      doc.content
+    end
+
+    def self.process(doc)
+      filters = ::Boilerpipe::Filters
+      # merge adjacent blocks with equal text_density
+      filters::SimpleBlockFusionProcessor.process doc
+
+      # merge text blocks next to each other
+      filters::BlockProximityFusion::MAX_DISTANCE_1.process doc
+
+      # marks text blocks as content / non-content using boilerpipe alg
+      filters::DensityRulesClassifier.process doc
+
+      doc
+    end
+  end
+end

data/lib/boilerpipe/filters/density_rules_classifier.rb

@@ -0,0 +1,39 @@
+# Classifies TextBlocks as content/not-content through rules that have been determined
+# using the C4.8 machine learning algorithm, as described in the paper
+# "Boilerplate Detection using Shallow Text Features", particularly using text densities and link
+# densities.
+
+module Boilerpipe::Filters
+  class DensityRulesClassifier
+
+    def self.process(doc)
+      #return doc if doc.text_blocks.size < 2
+
+      empty = Boilerpipe::Document::TextBlock.empty_start
+      text_blocks = [empty] + doc.text_blocks + [empty]
+
+      text_blocks.each_cons(3) do |slice|
+        prev, current, nxt = *slice
+        current.content = classify(prev, current, nxt)
+      end
+
+      doc
+    end
+
+    def self.classify(prev, current, nxt)
+      return false if current.link_density > 0.333333
+
+      if prev.link_density <= 0.555556
+        if current.text_density <= 9
+          return true if nxt.text_density > 10
+          return prev.text_density <= 4 ? false : true
+        else
+          return nxt.text_density == 0 ? false : true
+        end
+      else
+        return false if nxt.text_density <= 11
+        true
+      end
+    end
+  end
+end

data/lib/boilerpipe/filters/simple_block_fusion_processor.rb

@@ -0,0 +1,24 @@
+ # Merges two subsequent blocks if their text densities are equal.
+
+module Boilerpipe::Filters
+  class SimpleBlockFusionProcessor
+    def self.process(doc)
+      tbs = doc.text_blocks
+      return doc if tbs.size < 2
+
+      blocks_to_remove = []
+      tb1 = tbs.first
+      tbs.drop(1).each do |tb|
+        if tb1.text_density == tb.text_density
+          tb1.merge_next(tb)
+          blocks_to_remove << tb
+        else
+          tb1 = tb
+        end
+      end
+
+      doc.replace_text_blocks!( tbs - blocks_to_remove )
+      doc
+    end
+  end
+end

data/lib/boilerpipe/version.rb

@@ -1,3 +1,3 @@
 module Boilerpipe
-  VERSION = '0.1.1'
+  VERSION = '0.2.0'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: boilerpipe-ruby
 version: !ruby/object:Gem::Version
-  version: 0.1.1
+  version: 0.2.0
 platform: ruby
 authors:
 - Gregory Ostermayr
@@ -52,18 +52,32 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '3.0'
+- !ruby/object:Gem::Dependency
+  name: rickshaw
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.4.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.4.0
 - !ruby/object:Gem::Dependency
   name: nokogiri
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '='
+    - - ">="
       - !ruby/object:Gem::Version
         version: 1.6.6.2
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '='
+    - - ">="
       - !ruby/object:Gem::Version
         version: 1.6.6.2
 description: A pure ruby implementation of the boilerpipe algorithm
@@ -73,9 +87,12 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
+- ".circleci/config.yml"
 - ".gitignore"
 - ".rspec"
+- CHANGELOG.md
 - Gemfile
+- LICENSE.txt
 - README.md
 - Rakefile
 - bin/console
@@ -86,8 +103,10 @@ files:
 - lib/boilerpipe/document/text_document.rb
 - lib/boilerpipe/errors.rb
 - lib/boilerpipe/extractors/article_extractor.rb
+- lib/boilerpipe/extractors/default_extractor.rb
 - lib/boilerpipe/filters/block_proximity_fusion.rb
 - lib/boilerpipe/filters/boilerplate_block_filter.rb
+- lib/boilerpipe/filters/density_rules_classifier.rb
 - lib/boilerpipe/filters/document_title_match_classifier.rb
 - lib/boilerpipe/filters/expand_title_to_content_filter.rb
 - lib/boilerpipe/filters/heuristic_filter_base.rb
@@ -96,6 +115,7 @@ files:
 - lib/boilerpipe/filters/large_block_same_tag_level_to_content_filter.rb
 - lib/boilerpipe/filters/list_at_end_filter.rb
 - lib/boilerpipe/filters/num_words_rules_classifier.rb
+- lib/boilerpipe/filters/simple_block_fusion_processor.rb
 - lib/boilerpipe/filters/terminating_blocks_finder.rb
 - lib/boilerpipe/filters/trailing_headline_to_boilerplate_filter.rb
 - lib/boilerpipe/labels/default.rb
@@ -117,7 +137,8 @@ files:
 - lib/boilerpipe/version.rb
 - stuff.txt
 homepage: https://github.com/gregors/boilerpipe-ruby
-licenses: []
+licenses:
+- Apache 2.0
 metadata: {}
 post_install_message: 
 rdoc_options: []