curation 1.8 → 1.9

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: b8086e74a861e147bce3a73dcec3cebef7d5d98ede440c911f57757feaaac354
-  data.tar.gz: 3d6ff271bcfbf8599a76653d60b2c26ed3832c1c174885d114912f95a7707ee7
+  metadata.gz: e68898021d1c54927e120e46d2b1282535aec94a4daa0c552d9edeb4d6dc1d88
+  data.tar.gz: d6ba0ccabe71d10efb60fb0a5806f94f3fdd42f7e49fe39f00d64c427455d450
 SHA512:
-  metadata.gz: '084c4c3fbdf3491530cd4ca6ca0fdc736df69a22a49cce40b0d7c8d169f6a45e7c5f01f280957016e32bedca93d593bb2d9e60141e148ee4f110935877543168'
-  data.tar.gz: 3fe95dde59a8cb2268a1f93ff1e8b7ccbca9212a7b0cdc0093c16925549545df422945db25ab29f250fcc05242b5d7b01eb822f3c97b4e2b96d19653d70dff39
+  metadata.gz: 88310bda3ba8221af689f5848c24af2dfd48668caaf875b55f05626e09dbf072747b877e779b93a91c4e602b39cd23e61dc602662bb2b34e8b0e095c6cd9c488
+  data.tar.gz: c750a811383b541beefd3214accbdacda85ab7552ded3863734101cf1ffa42ec963b455c83f817e6176a2e31dc8d57a07ce14fac0f2567a54fd353c42f26d7fb

data/Gemfile.lock

@@ -1,7 +1,8 @@
 PATH
   remote: .
   specs:
-    curation (1.8)
+    curation (1.9)
+      htmlentities
       metainspector
       nokogiri
 
@@ -15,7 +16,7 @@ GEM
     byebug (11.1.3)
     domain_name (0.5.20190701)
       unf (>= 0.0.5, < 1.0.0)
-    faraday (1.9.3)
+    faraday (1.10.0)
       faraday-em_http (~> 1.0)
       faraday-em_synchrony (~> 1.0)
       faraday-excon (~> 1.1)
@@ -48,11 +49,12 @@ GEM
     faraday_middleware (1.2.0)
       faraday (~> 1.0)
     fastimage (2.2.6)
+    htmlentities (4.3.4)
     http-cookie (1.0.4)
       domain_name (~> 0.5)
-    metainspector (5.11.2)
+    metainspector (5.12.1)
       addressable (~> 2.7)
-      faraday (~> 1.4)
+      faraday (>= 1.4, < 3.0)
       faraday-cookie_jar (~> 0.0)
       faraday-encoding (~> 0.0)
       faraday-http-cache (~> 2.2)
@@ -60,7 +62,7 @@ GEM
       fastimage (~> 2.2)
       nesty (~> 1.0)
       nokogiri (~> 1.11)
-    mini_portile2 (2.7.1)
+    mini_portile2 (2.8.0)
     minitest (5.15.0)
     minitest-reporters (1.5.0)
       ansi
@@ -69,17 +71,17 @@ GEM
       ruby-progressbar
     multipart-post (2.1.1)
     nesty (1.0.2)
-    nokogiri (1.13.1)
-      mini_portile2 (~> 2.7.0)
+    nokogiri (1.13.6)
+      mini_portile2 (~> 2.8.0)
       racc (~> 1.4)
-    public_suffix (4.0.6)
+    public_suffix (4.0.7)
     racc (1.6.0)
     rake (12.3.3)
     ruby-progressbar (1.11.0)
     ruby2_keywords (0.0.5)
     unf (0.1.4)
       unf_ext
-    unf_ext (0.0.8)
+    unf_ext (0.0.8.1)
 
 PLATFORMS
   ruby

data/curation.gemspec

@@ -23,4 +23,5 @@ Gem::Specification.new do |spec|
   spec.require_paths = ["lib"]
   spec.add_dependency "metainspector"
   spec.add_dependency "nokogiri"
+  spec.add_dependency "htmlentities"
 end

data/lib/curation/version.rb

@@ -1,3 +1,3 @@
 module Curation
-  VERSION = "1.8"
+  VERSION = "1.9"
 end

data/lib/curation.rb

@@ -1,6 +1,7 @@
 require "curation/version"
 require "metainspector"
 require "open-uri"
+require "htmlentities"
 
 module Curation
   class Error < StandardError; end
@@ -37,6 +38,7 @@ module Curation
     end
 
     def text
+      # require 'byebug'; byebug
       @text ||= find_text
     end
 
@@ -49,6 +51,8 @@ module Curation
     def find_title
       if json_ld.any?
         json_ld.each do |ld|
+          # require 'byebug'; byebug
+          ld = ld.first if ld.is_a?(Array)
           return ld['headline'] if ld.has_key? 'headline'
         end
       end
@@ -70,6 +74,7 @@ module Curation
     def find_image
       if json_ld.any?
         json_ld.each do |ld|
+          ld = ld.first if ld.is_a?(Array)
           if ld.has_key? 'image'
             image_data = ld['image']
             return image_data if image_data.is_a? String
@@ -96,6 +101,14 @@ module Curation
     end
 
     def find_text
+      text = find_text_with_json_ld || find_text_with_nokogiri
+      text.to_s.gsub!('<br><br>', '<br>')
+      # require 'byebug'; byebug
+      text = clean_encoding text
+      text
+    end
+
+    def find_text_with_json_ld
       if json_ld.any?
         json_ld.each do |ld|
           next unless ['NewsArticle', 'ReportageNewsArticle'].include? ld['@type']
@@ -103,6 +116,10 @@ module Curation
           return ld['articleBody'] if ld.has_key? 'articleBody'
         end
       end
+      nil
+    end
+
+    def find_text_with_nokogiri
       h = nokogiri.dup
       BLACKLIST.each do |tag|
         h.css(tag).remove
@@ -110,7 +127,6 @@ module Curation
       nodes = h.css('p')
       nodes.xpath('//style').remove
       text = nodes.to_html
-      text.gsub!('<br><br>', '<br>')
       text
     end
 
@@ -125,6 +141,7 @@ module Curation
       return Date.parse metatags['pubdate'] rescue nil
       return Date.parse nokogiri.css('meta[property="article:published"]').first['content'] rescue nil
       return Date.parse nokogiri.css('meta[property="article:published_time"]').first['content'] rescue nil
+      return Date.parse nokogiri.css('meta[property="og:article:published_time"]').first['content'] rescue nil
       chunks = html.split('DisplayDate')
       if chunks.count > 1
         value = chunks[1]
@@ -156,13 +173,11 @@ module Curation
         begin
           options = nokogiri.css('[type="application/ld+json"]')
           options.each do |option|
-            # require 'byebug'; byebug
-            string = option.inner_text
-            hash = JSON.parse(string)
-            @json_ld << hash
+            @json_ld << json_ld_from_object(option)
           end
           # Some sites have tables in tables
           @json_ld.flatten!
+          # require 'byebug'; byebug
         rescue
           puts 'Curation::Page json_ld error'
         end
@@ -170,6 +185,12 @@ module Curation
       @json_ld
     end
 
+    def json_ld_from_object(object)
+      JSON.parse object.inner_text
+    rescue
+      {}
+    end
+
     def file
       @file ||= URI.open url, 'User-Agent' => "Mozilla/5.0"
     rescue
@@ -217,5 +238,26 @@ module Curation
     rescue
       puts 'Curation::Page metatags error'
     end
+
+    # r&Atilde;&copy;forme -> réforme
+    def clean_encoding(text)
+      clean_text = HTMLEntities.new.decode text
+      double_encoding = false
+      [
+        'é', # é
+        'è', # è
+        'î', # î
+        'ê', # ê
+      ].each do |string|
+        # require 'byebug'; byebug
+        double_encoding = true if clean_text.include? string
+      end
+      if double_encoding
+        clean_text.encode('iso-8859-1', undef: :replace)
+                  .force_encoding('utf-8')
+      else
+        text
+      end
+    end
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: curation
 version: !ruby/object:Gem::Version
-  version: '1.8'
+  version: '1.9'
 platform: ruby
 authors:
 - Arnaud Levy
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2022-02-14 00:00:00.000000000 Z
+date: 2022-05-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: metainspector
@@ -38,6 +38,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: htmlentities
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 description: When you build content curation tools, you need to extract the content
   of pages (title, text, image...). This requires different strategies and some fine
   tuning to work efficiently.