chupa-text-decomposer-html 1.0.2 → 1.0.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: '068db7f654ca70a8f65209c3d2f0d2fa1edf45ff'
-  data.tar.gz: 715d8630a7e69c100ed38a8bb090c5282b62fe2d
+  metadata.gz: 5b340a505f73aa5bcc6613a55b6fcec5e7988bc4
+  data.tar.gz: 3f4e1ef2b30decee069f76521753f6533a51024b
 SHA512:
-  metadata.gz: ab46f697f57427a940bd7968391648f54e2edbfab77974bc651ebe155f8dbfb6bbb9f8922d7153ee306ccb0402d30957a05fefe839437ba3ff28037f9a9e6ab0
-  data.tar.gz: 932251f709b54256f6478d4e7103f1b483c9c83166a277c6ee4b67b24c1528389ef1602ec960536de9b727cfcfa771e323d3467e9573911486f752e715ca5fcb
+  metadata.gz: 51f94a9bd3eb45765aa4518f8415ec82ec235f53e99e75e4ec94afb45721869587e9ad558fc7bdfeb05fca39ed76f24d08d770428b0a775c613f8b0768e60b7c
+  data.tar.gz: 409267fc2e80bc9cbc119443b6dee928ab72a7e1eb747c69cb8c781654fcd6977cf2d16663622c29f46e6c86923da0a9e8f911b82f73c8cb63e0722e9a130cda

data/chupa-text-decomposer-html.gemspec

@@ -22,7 +22,7 @@ end
 
 Gem::Specification.new do |spec|
   spec.name = "chupa-text-decomposer-html"
-  spec.version = "1.0.2"
+  spec.version = "1.0.3"
   spec.homepage = "https://github.com/ranguba/chupa-text-decomposer-html"
   spec.authors = ["Kouhei Sutou"]
   spec.email = ["kou@clear-code.com"]

data/doc/text/news.md

@@ -1,14 +1,26 @@
 # News
 
+## 1.0.3: 2017-07-10
+
+### Improvements
+
+  * Supported ignoring topic path content.
+
+  * Supported ignoring aside content.
+
+### Fixes
+
+  * Fixed a infinite loop bug.
+
 ## 1.0.2: 2017-07-05
 
-  * Support content based HTML detection.
+  * Supported content based HTML detection.
 
-  * Ignore common contents.
+  * Supported ignoring common contents.
 
 ## 1.0.1: 2014-02-18
 
-  * Support chupa-text 1.0.4.
+  * Supported chupa-text 1.0.4.
 
 ## 1.0.0: 2014-01-05
 

data/lib/chupa-text/decomposers/html.rb

@@ -28,6 +28,10 @@ module ChupaText
         "application/xhtml+xml",
       ]
       def target?(data)
+        (data["source-mime-types"] || []).each do |source_mime_type|
+          return false if TARGET_MIME_TYPES.include?(source_mime_type)
+        end
+
         return true if TARGET_EXTENSIONS.include?(data.extension)
         return true if TARGET_MIME_TYPES.include?(data.mime_type)
 
@@ -112,6 +116,7 @@ module ChupaText
         return text if header_element?(element, name, classes)
         return text if footer_element?(element, name, classes)
         return text if navigation_element?(element, name, classes)
+        return text if aside_element?(element, name, classes)
 
         element.children.each do |child|
           case child
@@ -143,7 +148,7 @@ module ChupaText
 
       def header_element?(element, name, classes)
         case name
-        when "header", "nav"
+        when "header"
           return true
         end
 
@@ -191,13 +196,34 @@ module ChupaText
 
         classes.each do |klass|
           case klass
-          when "nav", "menu"
+          when "nav", "menu", /\Atopic[-_]?path\z/i
+            return true
+          end
+        end
+
+        case element["id"]
+        when "nav", "menu", /\Atopic[-_]?path\z/i
+          return true
+        end
+
+        false
+      end
+
+      def aside_element?(element, name, classes)
+        case name
+        when "aside"
+          return true
+        end
+
+        classes.each do |klass|
+          case klass
+          when "aside"
             return true
           end
         end
 
         case element["id"]
-        when "nav", "menu"
+        when "aside"
           return true
         end
 

data/test/test-html.rb

@@ -28,6 +28,30 @@ class TestHTML < Test::Unit::TestCase
   end
 
   sub_test_case("target?") do
+    sub_test_case("source-mime-type") do
+      def create_data(uri, mime_type)
+        data = ChupaText::Data.new
+        data.body = ""
+        data.uri = uri
+        data["source-mime-types"] = [mime_type]
+        data
+      end
+
+      def test_text_html
+        data = create_data("index.html", "text/html")
+        assert do
+          not @decomposer.target?(data)
+        end
+      end
+
+      def test_application_xhtml_xml
+        data = create_data("index.html", "application/xhtml+xml")
+        assert do
+          not @decomposer.target?(data)
+        end
+      end
+    end
+
     sub_test_case("extension") do
       def create_data(uri)
         data = ChupaText::Data.new
@@ -436,6 +460,23 @@ class TestHTML < Test::Unit::TestCase
                        decompose(@data))
         end
 
+        def test_topic_path_class
+          @data.body = <<-HTML
+<html>
+  <body>
+    Before
+    <div class="topic-path">topic-path</div>
+    <div class="topic_path">topic_path</div>
+    <div class="topicpath">topicpath</div>
+    <div class="TopicPath">TopicPath</div>
+    After
+  </body>
+</html>
+          HTML
+          assert_equal(["Before\nAfter"],
+                       decompose(@data))
+        end
+
         def test_nav_id
           @data.body = <<-HTML
 <html>
@@ -450,6 +491,55 @@ class TestHTML < Test::Unit::TestCase
           @data.body = <<-HTML
 <html>
   <body>Before<div id="menu">nav</div>After</body>
+</html>
+          HTML
+          assert_equal(["BeforeAfter"],
+                       decompose(@data))
+        end
+
+        def test_topic_path_id
+          @data.body = <<-HTML
+<html>
+  <body>
+    Before
+    <div id="topic-path">topic-path</div>
+    <div id="topic_path">topic_path</div>
+    <div id="topicpath">topicpath</div>
+    <div id="TopicPath">TopicPath</div>
+    After
+  </body>
+</html>
+          HTML
+          assert_equal(["Before\nAfter"],
+                       decompose(@data))
+        end
+      end
+
+      sub_test_case("aside") do
+        def test_aside_tag
+          @data.body = <<-HTML
+<html>
+  <body>Before<aside>aside</aside>After</body>
+</html>
+          HTML
+          assert_equal(["BeforeAfter"],
+                       decompose(@data))
+        end
+
+        def test_aside_class
+          @data.body = <<-HTML
+<html>
+  <body>Before<div class="aside">aside</div>After</body>
+</html>
+          HTML
+          assert_equal(["BeforeAfter"],
+                       decompose(@data))
+        end
+
+        def test_aside_id
+          @data.body = <<-HTML
+<html>
+  <body>Before<div id="aside">aside</div>After</body>
 </html>
           HTML
           assert_equal(["BeforeAfter"],

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: chupa-text-decomposer-html
 version: !ruby/object:Gem::Version
-  version: 1.0.2
+  version: 1.0.3
 platform: ruby
 authors:
 - Kouhei Sutou
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-07-05 00:00:00.000000000 Z
+date: 2017-07-10 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: chupa-text