chupa-text-decomposer-html 1.0.2 → 1.0.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: '068db7f654ca70a8f65209c3d2f0d2fa1edf45ff'
-  data.tar.gz: 715d8630a7e69c100ed38a8bb090c5282b62fe2d
+SHA256:
+  metadata.gz: d18136db785fa061ef0bea3f17f8826a1ff55ed5020a591926e1700b91b9df38
+  data.tar.gz: 2c173149ac68d34756944ce98caa32f1dbc5bba4f86dab0461505bb90a5d406f
 SHA512:
-  metadata.gz: ab46f697f57427a940bd7968391648f54e2edbfab77974bc651ebe155f8dbfb6bbb9f8922d7153ee306ccb0402d30957a05fefe839437ba3ff28037f9a9e6ab0
-  data.tar.gz: 932251f709b54256f6478d4e7103f1b483c9c83166a277c6ee4b67b24c1528389ef1602ec960536de9b727cfcfa771e323d3467e9573911486f752e715ca5fcb
+  metadata.gz: 692141e0ed3d3d92729de8c47d62fa78ad6bc571070d293cdd8a7865e2d2366d82d68ef83121300dd92a82193b3db3ca83b7865ad6e0212c7e1b26e698830b13
+  data.tar.gz: 061e659f770c63f304cc7b697b6f3c512ea0c975eac35d55a9de9ad434820f056477275e4a240e058a42ace0187b1240290050bf83984049593003e309694411

data/Gemfile

@@ -1,6 +1,6 @@
-# -*- mode: ruby; coding: utf-8 -*-
+# -*- mode: ruby -*-
 #
-# Copyright (C) 2013  Kouhei Sutou <kou@clear-code.com>
+# Copyright (C) 2013-2024  Sutou Kouhei <kou@clear-code.com>
 #
 # This library is free software; you can redistribute it and/or
 # modify it under the terms of the GNU Lesser General Public
@@ -20,6 +20,12 @@ source "https://rubygems.org/"
 
 gemspec
 
+gem "bundler"
+gem "packnga"
+gem "rake"
+gem "redcarpet"
+gem "test-unit"
+
 base_dir = File.dirname(__FILE__)
 local_chupa_text_dir = File.join(base_dir, "..", "chupa-text")
 if File.exist?(local_chupa_text_dir)

data/chupa-text-decomposer-html.gemspec

@@ -1,6 +1,6 @@
 # -*- ruby -*-
 #
-# Copyright (C) 2013-2017  Kouhei Sutou <kou@clear-code.com>
+# Copyright (C) 2013-2024  Sutou Kouhei <kou@clear-code.com>
 #
 # This library is free software; you can redistribute it and/or
 # modify it under the terms of the GNU Lesser General Public
@@ -22,7 +22,7 @@ end
 
 Gem::Specification.new do |spec|
   spec.name = "chupa-text-decomposer-html"
-  spec.version = "1.0.2"
+  spec.version = "1.0.4"
   spec.homepage = "https://github.com/ranguba/chupa-text-decomposer-html"
   spec.authors = ["Kouhei Sutou"]
   spec.email = ["kou@clear-code.com"]
@@ -41,10 +41,4 @@ Gem::Specification.new do |spec|
 
   spec.add_runtime_dependency("chupa-text")
   spec.add_runtime_dependency("nokogiri")
-
-  spec.add_development_dependency("bundler")
-  spec.add_development_dependency("rake")
-  spec.add_development_dependency("test-unit")
-  spec.add_development_dependency("packnga")
-  spec.add_development_dependency("redcarpet")
 end

data/doc/text/news.md

@@ -1,14 +1,32 @@
 # News
 
+## 1.0.4: 2024-09-22
+
+### Improvements
+
+  * Removed NKF dependency.
+
+## 1.0.3: 2017-07-10
+
+### Improvements
+
+  * Supported ignoring topic path content.
+
+  * Supported ignoring aside content.
+
+### Fixes
+
+  * Fixed a infinite loop bug.
+
 ## 1.0.2: 2017-07-05
 
-  * Support content based HTML detection.
+  * Supported content based HTML detection.
 
-  * Ignore common contents.
+  * Supported ignoring common contents.
 
 ## 1.0.1: 2014-02-18
 
-  * Support chupa-text 1.0.4.
+  * Supported chupa-text 1.0.4.
 
 ## 1.0.0: 2014-01-05
 

data/lib/chupa-text/decomposers/html.rb

@@ -1,4 +1,4 @@
-# Copyright (C) 2013-2017  Kouhei Sutou <kou@clear-code.com>
+# Copyright (C) 2013-2024  Sutou Kouhei <kou@clear-code.com>
 #
 # This library is free software; you can redistribute it and/or
 # modify it under the terms of the GNU Lesser General Public
@@ -14,7 +14,6 @@
 # License along with this library; if not, write to the Free Software
 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
 
-require "nkf"
 require "nokogiri"
 
 module ChupaText
@@ -28,6 +27,10 @@ module ChupaText
         "application/xhtml+xml",
       ]
       def target?(data)
+        (data["source-mime-types"] || []).each do |source_mime_type|
+          return false if TARGET_MIME_TYPES.include?(source_mime_type)
+        end
+
         return true if TARGET_EXTENSIONS.include?(data.extension)
         return true if TARGET_MIME_TYPES.include?(data.mime_type)
 
@@ -45,7 +48,7 @@ module ChupaText
         doc = Nokogiri::HTML.parse(html, nil, guess_encoding(html))
         body_element = (doc % "body")
         if body_element
-          body = extract_text(body_element, "").scrub.gsub(/^\s+|\s+$/, '')
+          body = extract_text(body_element, +"").scrub.gsub(/^\s+|\s+$/, '')
         else
           body = ""
         end
@@ -85,7 +88,7 @@ module ChupaText
           if text.encoding != Encoding::ASCII_8BIT and text.valid_encoding?
             text.encoding.to_s
           else
-            guess_encoding_nkf(text)
+            guess_encoding_heuristic(text)
           end
         end
       end
@@ -101,8 +104,22 @@ module ChupaText
         end
       end
 
-      def guess_encoding_nkf(text)
-        NKF.guess(text).name
+      def guess_encoding_heuristic(text)
+        candidates = [
+          Encoding::EUC_JP,
+          Encoding::WINDOWS_31J,
+          Encoding::UTF16_BE,
+          Encoding::UTF16_LE,
+        ]
+        candidates.each do |candidate|
+          begin
+            text.encode(Encoding::UTF_8, candidate)
+          rescue EncodingError
+          else
+            return candidate.name
+          end
+        end
+        "UTF-8"
       end
 
       def extract_text(element, text)
@@ -112,6 +129,7 @@ module ChupaText
         return text if header_element?(element, name, classes)
         return text if footer_element?(element, name, classes)
         return text if navigation_element?(element, name, classes)
+        return text if aside_element?(element, name, classes)
 
         element.children.each do |child|
           case child
@@ -143,7 +161,7 @@ module ChupaText
 
       def header_element?(element, name, classes)
         case name
-        when "header", "nav"
+        when "header"
           return true
         end
 
@@ -191,13 +209,34 @@ module ChupaText
 
         classes.each do |klass|
           case klass
-          when "nav", "menu"
+          when "nav", "menu", /\Atopic[-_]?path\z/i
+            return true
+          end
+        end
+
+        case element["id"]
+        when "nav", "menu", /\Atopic[-_]?path\z/i
+          return true
+        end
+
+        false
+      end
+
+      def aside_element?(element, name, classes)
+        case name
+        when "aside"
+          return true
+        end
+
+        classes.each do |klass|
+          case klass
+          when "aside"
             return true
           end
         end
 
         case element["id"]
-        when "nav", "menu"
+        when "aside"
           return true
         end
 

data/test/run-test.rb

@@ -26,4 +26,4 @@ require "chupa-text"
 
 ChupaText::Decomposers.load
 
-exit(Test::Unit::AutoRunner.run(true))
+exit(Test::Unit::AutoRunner.run(true, __dir__))

data/test/test-html.rb

@@ -1,4 +1,4 @@
-# Copyright (C) 2013-2017  Kouhei Sutou <kou@clear-code.com>
+# Copyright (C) 2013-2024  Sutou Kouhei <kou@clear-code.com>
 #
 # This library is free software; you can redistribute it and/or
 # modify it under the terms of the GNU Lesser General Public
@@ -28,6 +28,30 @@ class TestHTML < Test::Unit::TestCase
   end
 
   sub_test_case("target?") do
+    sub_test_case("source-mime-type") do
+      def create_data(uri, mime_type)
+        data = ChupaText::Data.new
+        data.body = ""
+        data.uri = uri
+        data["source-mime-types"] = [mime_type]
+        data
+      end
+
+      def test_text_html
+        data = create_data("index.html", "text/html")
+        assert do
+          not @decomposer.target?(data)
+        end
+      end
+
+      def test_application_xhtml_xml
+        data = create_data("index.html", "application/xhtml+xml")
+        assert do
+          not @decomposer.target?(data)
+        end
+      end
+    end
+
     sub_test_case("extension") do
       def create_data(uri)
         data = ChupaText::Data.new
@@ -164,7 +188,7 @@ class TestHTML < Test::Unit::TestCase
 
       sub_test_case("detect") do
         def test_nothing
-          @data.body = <<-HTML.force_encoding("UTF-8")
+          @data.body = <<-HTML.dup.force_encoding("UTF-8")
 <html>
   <body>Hello</body>
 </html>
@@ -436,6 +460,23 @@ class TestHTML < Test::Unit::TestCase
                        decompose(@data))
         end
 
+        def test_topic_path_class
+          @data.body = <<-HTML
+<html>
+  <body>
+    Before
+    <div class="topic-path">topic-path</div>
+    <div class="topic_path">topic_path</div>
+    <div class="topicpath">topicpath</div>
+    <div class="TopicPath">TopicPath</div>
+    After
+  </body>
+</html>
+          HTML
+          assert_equal(["Before\nAfter"],
+                       decompose(@data))
+        end
+
         def test_nav_id
           @data.body = <<-HTML
 <html>
@@ -450,6 +491,55 @@ class TestHTML < Test::Unit::TestCase
           @data.body = <<-HTML
 <html>
   <body>Before<div id="menu">nav</div>After</body>
+</html>
+          HTML
+          assert_equal(["BeforeAfter"],
+                       decompose(@data))
+        end
+
+        def test_topic_path_id
+          @data.body = <<-HTML
+<html>
+  <body>
+    Before
+    <div id="topic-path">topic-path</div>
+    <div id="topic_path">topic_path</div>
+    <div id="topicpath">topicpath</div>
+    <div id="TopicPath">TopicPath</div>
+    After
+  </body>
+</html>
+          HTML
+          assert_equal(["Before\nAfter"],
+                       decompose(@data))
+        end
+      end
+
+      sub_test_case("aside") do
+        def test_aside_tag
+          @data.body = <<-HTML
+<html>
+  <body>Before<aside>aside</aside>After</body>
+</html>
+          HTML
+          assert_equal(["BeforeAfter"],
+                       decompose(@data))
+        end
+
+        def test_aside_class
+          @data.body = <<-HTML
+<html>
+  <body>Before<div class="aside">aside</div>After</body>
+</html>
+          HTML
+          assert_equal(["BeforeAfter"],
+                       decompose(@data))
+        end
+
+        def test_aside_id
+          @data.body = <<-HTML
+<html>
+  <body>Before<div id="aside">aside</div>After</body>
 </html>
           HTML
           assert_equal(["BeforeAfter"],

metadata

@@ -1,14 +1,13 @@
 --- !ruby/object:Gem::Specification
 name: chupa-text-decomposer-html
 version: !ruby/object:Gem::Version
-  version: 1.0.2
+  version: 1.0.4
 platform: ruby
 authors:
 - Kouhei Sutou
-autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-07-05 00:00:00.000000000 Z
+date: 2024-09-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: chupa-text
@@ -38,76 +37,6 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-- !ruby/object:Gem::Dependency
-  name: bundler
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-- !ruby/object:Gem::Dependency
-  name: rake
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-- !ruby/object:Gem::Dependency
-  name: test-unit
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-- !ruby/object:Gem::Dependency
-  name: packnga
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-- !ruby/object:Gem::Dependency
-  name: redcarpet
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
 description: |
   This is a ChupaText decomposer plugin for to extract text and
   meta-data from HTML.
@@ -133,7 +62,6 @@ homepage: https://github.com/ranguba/chupa-text-decomposer-html
 licenses:
 - LGPL-2.1+
 metadata: {}
-post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
@@ -148,9 +76,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.5.2
-signing_key: 
+rubygems_version: 3.6.0.dev
 specification_version: 4
 summary: This is a ChupaText decomposer plugin for to extract text and meta-data from
   HTML.