curation 2.0 → 2.0.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 8bfaa5cfe719f5a5f5c9f153a358e74f11eb4217f4bbddca880b8f45829fc913
-  data.tar.gz: 4abb219144b696da0f33ab12b251036bd67c4ca7250a4576febdcaaa8e409dc0
+  metadata.gz: f4c4661f04dfa6c3442537bf91ded409452311936ec3e2dedce8fa48ac6ba5d1
+  data.tar.gz: d48f91295be205e59d3028907f1f6dfa9e0d83bc5322834b4b8b21ce824e62a2
 SHA512:
-  metadata.gz: 6d19e2a775ea069a3e6b0dc50b66f30d52f0c4e1e101cea2c9f0dde5cedde611043cf4267a4764994be8105e422919bf708f850935d531d9bb8f907025393a05
-  data.tar.gz: 4f0dd67b073cc2003cf798603f7221ba32977645180b671bcde0b1464fc53085cf9a076c2f30f4ce3775a8524246386d576e1bd0d1021c8388db58d13c16068f
+  metadata.gz: dbcc3db9b67f0b91773a1e72626d17c9e3ded02054f6b348bc5264005c09518d568e851f080e780316e32455941ee92146daa35c61c6fc3866806d698a949672
+  data.tar.gz: 0c03a44f6f97cf5f0180db1832989c9e0eadd0e7250d3f44a5c4cd6a10d9f42d437081a53449fe40a2c63b4e4b398eedc763fd29f7e3173bab3ee10fd7ca157a

data/Gemfile

@@ -7,3 +7,4 @@ gem 'rake', '~> 12.0'
 gem 'minitest'
 gem 'minitest-reporters'
 gem 'byebug'
+

data/Gemfile.lock

@@ -1,10 +1,11 @@
 PATH
   remote: .
   specs:
-    curation (2.0)
+    curation (2.0.2)
       htmlentities
       metainspector
       nokogiri
+      rails-html-sanitizer
 
 GEM
   remote: https://rubygems.org/
@@ -15,6 +16,7 @@ GEM
     base64 (0.2.0)
     builder (3.2.4)
     byebug (11.1.3)
+    crass (1.0.6)
     domain_name (0.5.20190701)
       unf (>= 0.0.5, < 1.0.0)
     faraday (2.7.11)
@@ -40,6 +42,9 @@ GEM
     htmlentities (4.3.4)
     http-cookie (1.0.5)
       domain_name (~> 0.5)
+    loofah (2.21.4)
+      crass (~> 1.0.2)
+      nokogiri (>= 1.12.0)
     metainspector (5.15.0)
       addressable (~> 2.8.4)
       faraday (~> 2.5)
@@ -63,6 +68,9 @@ GEM
       racc (~> 1.4)
     public_suffix (5.0.3)
     racc (1.7.3)
+    rails-html-sanitizer (1.6.0)
+      loofah (~> 2.21)
+      nokogiri (~> 1.14)
     rake (12.3.3)
     ruby-progressbar (1.13.0)
     ruby2_keywords (0.0.5)

data/curation.gemspec

@@ -24,4 +24,5 @@ Gem::Specification.new do |spec|
   spec.add_dependency "metainspector"
   spec.add_dependency "nokogiri"
   spec.add_dependency "htmlentities"
+  spec.add_dependency "rails-html-sanitizer"
 end

data/lib/curation/finders/text.rb

@@ -1,6 +1,12 @@
 module Text
 
-  BLACKLIST = [
+  def text
+    @text ||= find_text_and_clean
+  end
+
+  protected
+
+  BLACKLIST_HARD = [
     'head', 'script', 'style', 'iframe', 'nav', 'noscript', 'header', 'footer', 'aside',
     '.navigation', '.top-menu-container', '.navbar', '.navbar-header', '.breadcrumb',
     '#breadcrumbs', '[typeof="v:Breadcrumb"]', '.skip-link', '.search', '.search-form',
@@ -11,19 +17,23 @@ module Text
     '[style*="display: none;"]', '[style*="display: none"]', '[aria-hidden="true"]'
   ]
 
-  def text
-    # require 'byebug'; byebug
-    @text ||= find_text
-  end
+  BLACKLIST_SOFT = [
+    'head', 'script', 'noscript', 'style', 'iframe', 'nav', 'footer', 'aside', '[role="dialog"]'
+  ]
 
-  protected
+  def find_text_and_clean
+    text = find_text.to_s.dup
+    text = text.gsub('<br><br>', '<br>')
+    text = text.gsub(/\s+/, ' ')
+    text = clean_encoding(text)
+    text = Rails::HTML5::FullSanitizer.new.sanitize(text)
+    text
+  end
 
   def find_text
-    text = find_text_with_json_ld || find_text_with_nokogiri
-    text.to_s.dup.gsub!('<br><br>', '<br>')
-    # require 'byebug'; byebug
-    text = clean_encoding text
-    text
+    find_text_with_json_ld || 
+    find_text_with_nokogiri_hard ||
+    find_text_with_nokogiri_soft
   end
 
   def find_text_with_json_ld
@@ -34,24 +44,27 @@ module Text
         return ld['articleBody'] if ld.has_key? 'articleBody'
       end
     end
-    nil
+    false
   end
 
-  def find_text_with_nokogiri
+  def find_text_with_nokogiri_hard
     h = nokogiri.dup
     h.xpath('//style').remove
-    BLACKLIST.each do |tag|
+    BLACKLIST_HARD.each do |tag|
       h.css(tag).remove
     end
     nodes = h.css('p')
-    if nodes.any? 
-      text = nodes.to_html
-      text
-    else
-      # Cleanup was too hard, let's try softer
-      h = nokogiri.dup
-      h.text
+    text = nodes.to_html
+    text.present? ? text : false
+  end
+
+  def find_text_with_nokogiri_soft
+    h = nokogiri.dup
+    h.xpath('//style').remove
+    BLACKLIST_SOFT.each do |tag|
+      h.css(tag).remove
     end
+    h.text
   end
 
   # r&Atilde;&copy;forme -> réforme
@@ -64,7 +77,6 @@ module Text
       'î', # î
       'ê', # ê
     ].each do |string|
-      # require 'byebug'; byebug
       double_encoding = true if clean_text.include? string
     end
     if double_encoding

data/lib/curation/finders/title.rb

@@ -21,7 +21,7 @@ module Title
         return ld['headline'] if ld.has_key? 'headline'
       end
     end
-    nil
+    false
   end
 
   def find_title_with_metainspector
@@ -39,6 +39,7 @@ module Title
     elsif metainspector_title.present?
       return metainspector_title
     end
+    false
   end
 
   def find_title_with_nokogiri

data/lib/curation/version.rb

@@ -1,3 +1,3 @@
 module Curation
-  VERSION = "2.0"
+  VERSION = "2.0.2"
 end

data/lib/curation.rb

@@ -10,6 +10,7 @@ require "curation/finders/title"
 require "metainspector"
 require "open-uri"
 require "htmlentities"
+require "rails-html-sanitizer"
 
 module Curation
   class Error < StandardError; end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: curation
 version: !ruby/object:Gem::Version
-  version: '2.0'
+  version: 2.0.2
 platform: ruby
 authors:
 - Arnaud Levy
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2023-11-11 00:00:00.000000000 Z
+date: 2023-11-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: metainspector
@@ -52,6 +52,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: rails-html-sanitizer
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 description: When you build content curation tools, you need to extract the content
   of pages (title, text, image...). This requires different strategies and some fine
   tuning to work efficiently.