curation 2.0 → 2.0.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 8bfaa5cfe719f5a5f5c9f153a358e74f11eb4217f4bbddca880b8f45829fc913
-  data.tar.gz: 4abb219144b696da0f33ab12b251036bd67c4ca7250a4576febdcaaa8e409dc0
+  metadata.gz: c2062c7ec7fb444d27f102d26658b386a01a964ea4e03aa0d81a472316012d11
+  data.tar.gz: 634f5216e61801b3ac42c8b340d5012c101024f4d5b5e0ba155c01a89f38cafb
 SHA512:
-  metadata.gz: 6d19e2a775ea069a3e6b0dc50b66f30d52f0c4e1e101cea2c9f0dde5cedde611043cf4267a4764994be8105e422919bf708f850935d531d9bb8f907025393a05
-  data.tar.gz: 4f0dd67b073cc2003cf798603f7221ba32977645180b671bcde0b1464fc53085cf9a076c2f30f4ce3775a8524246386d576e1bd0d1021c8388db58d13c16068f
+  metadata.gz: 15008b92c6a51fdf9bd79b9f1da01d59323c13589b6e8ccd5d8754b96153173fe8a2cdffa5faa45a4d38c92b097a35012f8dff98fea6244ade8c9af22c13d1cc
+  data.tar.gz: 96a29d3c8482fce0101f91a3f24eafaec51a9115f2169831b67939edf79a47d7f5bb0c3223466de355cf8a367143a262cc54a1d5e4dd7e6bffe2d310879a9cae

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    curation (2.0)
+    curation (2.0.1)
       htmlentities
       metainspector
       nokogiri

data/lib/curation/finders/text.rb

@@ -1,6 +1,12 @@
 module Text
 
-  BLACKLIST = [
+  def text
+    @text ||= find_text_and_clean
+  end
+
+  protected
+
+  BLACKLIST_HARD = [
     'head', 'script', 'style', 'iframe', 'nav', 'noscript', 'header', 'footer', 'aside',
     '.navigation', '.top-menu-container', '.navbar', '.navbar-header', '.breadcrumb',
     '#breadcrumbs', '[typeof="v:Breadcrumb"]', '.skip-link', '.search', '.search-form',
@@ -11,19 +17,23 @@ module Text
     '[style*="display: none;"]', '[style*="display: none"]', '[aria-hidden="true"]'
   ]
 
-  def text
-    # require 'byebug'; byebug
-    @text ||= find_text
-  end
+  BLACKLIST_SOFT = [
+    'head', 'script', 'noscript', 'style', 'iframe', 'nav', 'footer', 'aside', '[role="dialog"]'
+  ]
 
-  protected
+  def find_text_and_clean
+    text = find_text.to_s.dup
+    text = text.gsub('<br><br>', '<br>')
+    text = text.gsub(/\s+/, ' ')
+    text = clean_encoding(text)
+    text
+
+  end
 
   def find_text
-    text = find_text_with_json_ld || find_text_with_nokogiri
-    text.to_s.dup.gsub!('<br><br>', '<br>')
-    # require 'byebug'; byebug
-    text = clean_encoding text
-    text
+    find_text_with_json_ld || 
+    find_text_with_nokogiri_hard ||
+    find_text_with_nokogiri_soft
   end
 
   def find_text_with_json_ld
@@ -34,24 +44,27 @@ module Text
         return ld['articleBody'] if ld.has_key? 'articleBody'
       end
     end
-    nil
+    false
   end
 
-  def find_text_with_nokogiri
+  def find_text_with_nokogiri_hard
     h = nokogiri.dup
     h.xpath('//style').remove
-    BLACKLIST.each do |tag|
+    BLACKLIST_HARD.each do |tag|
       h.css(tag).remove
     end
     nodes = h.css('p')
-    if nodes.any? 
-      text = nodes.to_html
-      text
-    else
-      # Cleanup was too hard, let's try softer
-      h = nokogiri.dup
-      h.text
+    text = nodes.to_html
+    text.present? ? text : false
+  end
+
+  def find_text_with_nokogiri_soft
+    h = nokogiri.dup
+    h.xpath('//style').remove
+    BLACKLIST_SOFT.each do |tag|
+      h.css(tag).remove
     end
+    h.text
   end
 
   # r&Atilde;&copy;forme -> réforme
@@ -64,7 +77,6 @@ module Text
       'î', # î
       'ê', # ê
     ].each do |string|
-      # require 'byebug'; byebug
       double_encoding = true if clean_text.include? string
     end
     if double_encoding

data/lib/curation/finders/title.rb

@@ -21,7 +21,7 @@ module Title
         return ld['headline'] if ld.has_key? 'headline'
       end
     end
-    nil
+    false
   end
 
   def find_title_with_metainspector
@@ -39,6 +39,7 @@ module Title
     elsif metainspector_title.present?
       return metainspector_title
     end
+    false
   end
 
   def find_title_with_nokogiri

data/lib/curation/version.rb

@@ -1,3 +1,3 @@
 module Curation
-  VERSION = "2.0"
+  VERSION = "2.0.1"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: curation
 version: !ruby/object:Gem::Version
-  version: '2.0'
+  version: 2.0.1
 platform: ruby
 authors:
 - Arnaud Levy