boilerpipe-ruby 0.4.3 → 0.4.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: da8bc0b8d74eea14b73e61812bbeba5fef75e8bae2330739e49b28e26f73d14d
-  data.tar.gz: 68fee529b501210cf3278eb2b045b09e6d27c7846355b7d430c05e60f39088e2
+  metadata.gz: 835e5122c287082ef39874ee0a3365f08910a7a3f70b76c45dc264e8e1301edc
+  data.tar.gz: edff0860c01277adfc703e453e22b1d24c2db1817cfd4ad6800dfc1a6c0f339f
 SHA512:
-  metadata.gz: 4202afab2a01ae588977fde351dfacc29551634077e794d028282666c43d3aeb09adf425d60608dc694c36f2fd8ed034ef89ba41ee07e6ad23f426c19d740931
-  data.tar.gz: a0dc75a0c5384e1eaf8b50dfb92bc9294b41f38e8b0e0cceb9e9a6aafdf436629df2b3420f61a33523a08d6788c02ade799cdf8fe29db4338c766c7b01523704
+  metadata.gz: 7cbeb85f9dd74c930e9999e850d2abca6fd3e9470971461a978295de640195192f633a71a19a20b96824fb5becec2e7841e7f3e257b83b90550c935b6d16ff7b
+  data.tar.gz: 393803c95416f77a467724cd0b451961dcbb9c6e544c6d22a4f71853a041e5f2c32513ffd3f43e9733de43624fce31dad18b9cb7d0c37bd3cd722a6fdf223481

data/CHANGELOG.md

@@ -1,3 +1,6 @@
+# 0.4.4 / 2021-02-13
+* Do a better job of stripping out script tags
+
 # 0.4.3 / 2020-07-18
 
 * update deps

data/boilerpipe-ruby.gemspec

@@ -22,6 +22,6 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency 'bundler', '~> 2.0'
   spec.add_development_dependency 'rake', '>= 12.3.3'
   spec.add_development_dependency 'rickshaw', '~> 0.5.0'
-  spec.add_development_dependency 'rspec', '~> 3.9'
+  spec.add_development_dependency 'rspec', '~> 3.10'
   spec.add_runtime_dependency 'nokogiri', '~> 1.10'
 end

data/lib/boilerpipe.rb

@@ -40,6 +40,7 @@ require 'boilerpipe/filters/trailing_headline_to_boilerplate_filter'
 require 'boilerpipe/labels/default'
 require 'boilerpipe/labels/label_action'
 
+require 'boilerpipe/sax/preprocessor'
 require 'boilerpipe/sax/html_content_handler'
 require 'boilerpipe/sax/boilerpipe_html_parser'
 require 'boilerpipe/sax/tag_action_map'

data/lib/boilerpipe/sax/boilerpipe_html_parser.rb

@@ -1,16 +1,11 @@
 module Boilerpipe::SAX
   class BoilerpipeHTMLParser
     def self.parse(text)
-      # script bug - delete script tags
-      text.gsub!(/\<script>.+?<\/script>/i, '')
-
-      # nokogiri uses libxml for mri and nekohtml for jruby
-      # mri doesn't remove &nbsp; when missing the semicolon
-      text.gsub!(/(&nbsp) /, '\1; ')
+      # strip out tags that cause issues
+      text = Preprocessor.strip(text)
 
       # use nokogiri to fix any bad tags, errors - keep experimenting with this
       text = Nokogiri::HTML(text).to_html
-
       handler = HTMLContentHandler.new
       noko_parser = Nokogiri::HTML::SAX::Parser.new(handler)
       noko_parser.parse(text)

data/lib/boilerpipe/sax/preprocessor.rb

@@ -0,0 +1,11 @@
+module Boilerpipe::SAX
+  class Preprocessor
+    def self.strip(text)
+      # script bug - delete script tags
+      text = text.gsub(/\<script.+?<\/script>/im, '')
+      # nokogiri uses libxml for mri and nekohtml for jruby
+      # mri doesn't remove &nbsp; when missing the semicolon
+      text.gsub(/(&nbsp) /, '\1; ')
+    end
+  end
+end

data/lib/boilerpipe/version.rb

@@ -1,3 +1,3 @@
 module Boilerpipe
-  VERSION = '0.4.3'
+  VERSION = '0.4.4'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: boilerpipe-ruby
 version: !ruby/object:Gem::Version
-  version: 0.4.3
+  version: 0.4.4
 platform: ruby
 authors:
 - Gregory Ostermayr
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2020-07-22 00:00:00.000000000 Z
+date: 2021-02-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -58,14 +58,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.9'
+        version: '3.10'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.9'
+        version: '3.10'
 - !ruby/object:Gem::Dependency
   name: nokogiri
   requirement: !ruby/object:Gem::Requirement
@@ -136,6 +136,7 @@ files:
 - lib/boilerpipe/labels/label_action.rb
 - lib/boilerpipe/sax/boilerpipe_html_parser.rb
 - lib/boilerpipe/sax/html_content_handler.rb
+- lib/boilerpipe/sax/preprocessor.rb
 - lib/boilerpipe/sax/tag_action_map.rb
 - lib/boilerpipe/sax/tag_actions/anchor_text.rb
 - lib/boilerpipe/sax/tag_actions/block_level.rb