curation 1.1 → 1.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 14637da5bf7b047f9c34b40ff4bf108110dce7408d3c12af9e76222141bdcf65
-  data.tar.gz: 144487e448354476958895db783fdc681d0116ec3eca5e7ede1cd3cf1bd1d13a
+  metadata.gz: f059bacdf3a2deedf5363721c8186af8a62d5b69709022b3d076f725433be26f
+  data.tar.gz: ac36789448e1b3a58161f98362dbed1d889537612939f86d4a2db31b935811c3
 SHA512:
-  metadata.gz: 303babe96daee5e792672e1ece05b49d77c115e393af00ded6902b10b9d4adb51210923c7d6c0e68a359246078618a8d09606ab57dd57fa0882db7e3a474e3e4
-  data.tar.gz: 451a1ad8866adc91850ff9b150847564bb9de43b15432de2dbf06fb2936a3788ed2c0bf4d9e3c9c462e13bdb5c6236bf5dc0c97a6697a8c3ea9c307bf2f35410
+  metadata.gz: 311113b9b172cfb54f917b694ca5a8f2bb17184c38fc99c8930592f73de9219c9ecda9217562180ef0e17def15682f56255b5f7e671555071d32b9149479cc2b
+  data.tar.gz: bbbe41e2fce674677dac3f3d7c5b724c6bbde6e8941f8be79f8d695ce722578046cb061d7c25fcac771ecb1bb57223d4c88748f36c85c0d6d08cac354ecb6a40

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    curation (0.1.0)
+    curation (1.6)
       metainspector
       nokogiri
 

data/lib/curation.rb

@@ -19,17 +19,13 @@ module Curation
       '[style*="display: none;"]', '[style*="display: none"]', '[aria-hidden="true"]'
     ]
 
-    def initialize(url)
+    def initialize(url, html = nil)
       @url = url
+      @html = html
     end
 
     def title
-      if json_ld.any?
-        json_ld.each do |ld|
-          return ld['headline'] if ld.has_key? 'headline'
-        end
-      end
-      metainspector.best_title
+      @title = find_title
     end
 
     def image
@@ -46,7 +42,7 @@ module Curation
           return ld['articleBody'] if ld.has_key? 'articleBody'
         end
       end
-      h = html.dup
+      h = nokogiri.dup
       BLACKLIST.each do |tag|
         h.css(tag).remove
       end
@@ -59,53 +55,90 @@ module Curation
 
     protected
 
+    def find_title
+      if json_ld.any?
+        # Some sites have tables in tables
+        json_ld.flatten!
+        json_ld.each do |ld|
+          return ld['headline'] if ld.has_key? 'headline'
+        end
+      end
+      begin
+        [
+          metainspector.best_title,
+          metainspector.title,
+          nokogiri.css('[itemprop="headline"]')&.first&.inner_text,
+          nokogiri.css('title')&.first&.inner_text
+        ].each do |possibility|
+          return possibility unless possibility.blank?
+        end
+      rescue
+        puts 'Curation::Page find_title error'
+      end
+      return ''
+    end
+
     def find_image
       if json_ld.any?
         json_ld.each do |ld|
           if ld.has_key? 'image'
             image_data = ld['image']
             return image_data if image_data.is_a? String
-            return image_data.first if image_data.is_a? Array
+            if image_data.is_a? Array
+              first = image_data.first
+              return first if first.is_a? String
+              return first['url'] if first.is_a? Hash
+            end
             return image_data['url'] if image_data.is_a? Hash
           end
         end
       end
-      metainspector.images.best
-    end
-
-    def html
-      @html ||= Nokogiri::HTML data
-    rescue
-      puts "Nokogiri error"
+      begin
+        [
+          metainspector.images.best,
+          nokogiri.css('[property="og:image"]').first&.attributes['content'].value
+        ].each do |possibility|
+            return possibility unless possibility.blank?
+        end
+      rescue
+        puts 'Curation::Page find_image error'
+      end
+      return ''
     end
 
     def json_ld
       unless @json_ld
         @json_ld = []
         begin
-          options = html.css('[type="application/ld+json"]')
+          options = nokogiri.css('[type="application/ld+json"]')
           options.each do |option|
             string = option.inner_text
             hash = JSON.parse(string)
             @json_ld << hash
           end
         rescue
-          puts "JSON LD error"
+          puts 'Curation::Page json_ld error'
         end
       end
       @json_ld
     end
 
-    def data
-      URI.open url
+    def html
+      @html ||= URI.open url
     rescue
       puts "Impossible to open #{url}"
     end
 
+    def nokogiri
+      @nokogiri ||= Nokogiri::HTML html
+    rescue
+      puts 'Curation::Page nokogiri error'
+    end
+
     def metainspector
-      @metainspector ||= MetaInspector.new url
+      @metainspector ||= MetaInspector.new url, document: html
     rescue
-      puts "MetaInspector error"
+      puts 'Curation::Page metainspector error'
     end
   end
 end

data/lib/curation/version.rb

@@ -1,3 +1,3 @@
 module Curation
-  VERSION = "1.1"
+  VERSION = "1.6"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: curation
 version: !ruby/object:Gem::Version
-  version: '1.1'
+  version: '1.6'
 platform: ruby
 authors:
 - Arnaud Levy
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2020-07-20 00:00:00.000000000 Z
+date: 2020-07-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: metainspector