curation 1.2 → 1.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 21a638c75deffb70db8694ecb99d94b1b711fba922ecbd0826d3a69597031df6
-  data.tar.gz: 621519940fe6bc44302201f64211eba932d52ed366ebe0c6f11c02688b5ae0ba
+  metadata.gz: e3e4d559a2583393e2d58cba1bfe30ef8870733ce013c00780b29706b9bfce68
+  data.tar.gz: 5802c61d564cfc2568f9173f6b270e3e288f4dbdfcfec624b02205bd01a81aaf
 SHA512:
-  metadata.gz: bf5f63d05793d71ac052b94dcaf5f4e34943355799e042361ae704ca9423b501ab3e157ae8b6509da642e6fa680768205030a24ddc1eb4e24c31863284289575
-  data.tar.gz: d09140b59ba119e85e9c955a40586bd1ce6410fe392a59b5e4a92e2a1ae80b4608fd458c0b84b24be8ba31267bcee9550432264e6ca4068be47b90db46228ed1
+  metadata.gz: 7cd290b2f9b5d9188c9db0bd07a9f88ce564faac927c3590af5c88cf11be0a848dd209fd7c0dd48e5af103bc8f05822911e7ffbe94c9a35dab0732e9e7d70a21
+  data.tar.gz: 0bcd9304b78f9ddb362ef7809831fe5065454adc64aa117df302d8a0ec384d92d7aac64e2dc1e8ebc3caf12267d78dbf7150f9fe12254a2b1d31509644e1e62e

data/.gitignore

@@ -6,3 +6,4 @@
 /pkg/
 /spec/reports/
 /tmp/
+.byebug_history

data/Gemfile

@@ -3,4 +3,7 @@ source "https://rubygems.org"
 # Specify your gem's dependencies in curation.gemspec
 gemspec
 
-gem "rake", "~> 12.0"
+gem 'rake', '~> 12.0'
+gem 'minitest'
+gem 'minitest-reporters'
+gem 'byebug'

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    curation (0.1.0)
+    curation (1.7)
       metainspector
       nokogiri
 
@@ -10,6 +10,9 @@ GEM
   specs:
     addressable (2.7.0)
       public_suffix (>= 2.0.2, < 5.0)
+    ansi (1.5.0)
+    builder (3.2.4)
+    byebug (11.1.3)
     domain_name (0.5.20190701)
       unf (>= 0.0.5, < 1.0.0)
     faraday (1.0.1)
@@ -37,12 +40,19 @@ GEM
       nesty (~> 1.0.2)
       nokogiri (~> 1.10.9)
     mini_portile2 (2.4.0)
+    minitest (5.14.1)
+    minitest-reporters (1.4.2)
+      ansi
+      builder
+      minitest (>= 5.0)
+      ruby-progressbar
     multipart-post (2.1.1)
     nesty (1.0.2)
     nokogiri (1.10.10)
       mini_portile2 (~> 2.4.0)
     public_suffix (4.0.5)
     rake (12.3.3)
+    ruby-progressbar (1.10.1)
     unf (0.1.4)
       unf_ext
     unf_ext (0.0.7.7)
@@ -51,7 +61,10 @@ PLATFORMS
   ruby
 
 DEPENDENCIES
+  byebug
   curation!
+  minitest
+  minitest-reporters
   rake (~> 12.0)
 
 BUNDLED WITH

data/Rakefile

@@ -1,2 +1,10 @@
 require "bundler/gem_tasks"
-task :default => :spec
+require "rake/testtask"
+
+Rake::TestTask.new do |t|
+  t.libs << "test"
+  t.test_files = FileList["test/**/*_spec.rb"]
+  t.warning = false
+end
+
+task default: :test

data/lib/curation.rb

@@ -6,7 +6,7 @@ module Curation
   class Error < StandardError; end
 
   class Page
-    attr_reader :url, :title, :text, :image
+    attr_reader :url
 
     BLACKLIST = [
       'head', 'script', 'style', 'iframe', 'nav', 'noscript', 'header', 'footer', 'aside',
@@ -25,25 +25,80 @@ module Curation
     end
 
     def title
+      @title ||= find_title
+    end
+
+    def image
+      unless @image
+        @image = find_image
+        @image = @image.to_s.gsub('http://', 'https://')
+      end
+      @image
+    end
+
+    def text
+      @text ||= find_text
+    end
+
+    def date
+      @date ||= find_date
+    end
+
+    protected
+
+    def find_title
       if json_ld.any?
         json_ld.each do |ld|
           return ld['headline'] if ld.has_key? 'headline'
         end
       end
-      metainspector.best_title unless metainspector.best_title.blank?
-      metainspector.title
+      begin
+        [
+          metainspector.best_title,
+          metainspector.title,
+          nokogiri.css('[itemprop="headline"]')&.first&.inner_text,
+          nokogiri.css('title')&.first&.inner_text
+        ].each do |possibility|
+          return possibility unless possibility.to_s.empty?
+        end
+      rescue
+        puts 'Curation::Page find_title error'
+      end
+      return ''
     end
 
-    def image
-      @image = find_image
-      @image = @image.to_s.gsub('http://', 'https://')
-      @image
+    def find_image
+      if json_ld.any?
+        json_ld.each do |ld|
+          if ld.has_key? 'image'
+            image_data = ld['image']
+            return image_data if image_data.is_a? String
+            if image_data.is_a? Array
+              first = image_data.first
+              return first if first.is_a? String
+              return first['url'] if first.is_a? Hash
+            end
+            return image_data['url'] if image_data.is_a? Hash
+          end
+        end
+      end
+      begin
+        [
+          metainspector.images.best,
+          nokogiri.css('[property="og:image"]').first&.attributes['content'].value
+        ].each do |possibility|
+          return possibility unless possibility.to_s.empty?
+        end
+      rescue
+        puts 'Curation::Page find_image error'
+      end
+      return ''
     end
 
-    def text
+    def find_text
       if json_ld.any?
         json_ld.each do |ld|
-          next unless ld['@type'] == 'NewsArticle'
+          next unless ['NewsArticle', 'ReportageNewsArticle'].include? ld['@type']
           return ld['text'] if ld.has_key? 'text'
           return ld['articleBody'] if ld.has_key? 'articleBody'
         end
@@ -59,24 +114,44 @@ module Curation
       text
     end
 
-    protected
-
-    def find_image
+    def find_date
       if json_ld.any?
         json_ld.each do |ld|
-          if ld.has_key? 'image'
-            image_data = ld['image']
-            return image_data if image_data.is_a? String
-            return image_data.first if image_data.is_a? Array
-            return image_data['url'] if image_data.is_a? Hash
-          end
+          next unless ['NewsArticle', 'ReportageNewsArticle'].include? ld['@type']
+          return Date.parse ld['datePublished'] if ld.has_key? 'datePublished'
         end
       end
-      metainspector.images.best
+      return Date.parse metatags['date'] rescue nil
+      return Date.parse metatags['pubdate'] rescue nil
+      return Date.parse nokogiri.css('meta[property="article:published"]').first['content'] rescue nil
+      return Date.parse nokogiri.css('meta[property="article:published_time"]').first['content'] rescue nil
+      chunks = html.split('DisplayDate')
+      if chunks.count > 1
+        value = chunks[1]
+        value = value.split(',').first
+        value = value.gsub('"', '')
+        value = value[1..-1] if value[0] == ':'
+        return Date.parse value rescue nil
+      end
+      begin
+        value = nokogiri.css('.postDate').first
+        value = value.inner_text
+        value = value.gsub(' — ', '')
+        return Date.parse value
+      rescue
+      end
+      begin
+        value = nokogiri.css('.gta_post_date').first
+        value = value.inner_text
+        return Date.parse value
+      rescue
+      end
     end
 
+    private
+
     def json_ld
-      unless @json_ld
+      unless defined?(@json_ld)
         @json_ld = []
         begin
           options = nokogiri.css('[type="application/ld+json"]')
@@ -85,29 +160,57 @@ module Curation
             hash = JSON.parse(string)
             @json_ld << hash
           end
+          # Some sites have tables in tables
+          @json_ld.flatten!
         rescue
-          puts "JSON LD error"
+          puts 'Curation::Page json_ld error'
         end
       end
       @json_ld
     end
 
+    def file
+      @file ||= URI.open url, 'User-Agent' => "Mozilla/5.0"
+    rescue
+      puts "Curation::Page file error with url #{url}"
+    end
+
     def html
-      @html ||= URI.open url
+      unless @html
+        file.rewind
+        @html = file.read
+        file.rewind
+      end
+      @html
     rescue
-      puts "Impossible to open #{url}"
+      puts "Curation::Page html error"
     end
 
     def nokogiri
-      @nokogiri ||= Nokogiri::HTML html
+      unless @nokogiri
+        file.rewind
+        @nokogiri = Nokogiri::HTML file
+        file.rewind
+      end
+      @nokogiri
     rescue
-      puts "Nokogiri error"
+      puts 'Curation::Page nokogiri error'
     end
 
     def metainspector
-      @metainspector ||= MetaInspector.new url, document: html
+      unless @metainspector
+        @metainspector = html.nil?  ? MetaInspector.new(url)
+                                    : MetaInspector.new(url, document: html)
+      end
+      @metainspector
+    rescue
+      puts 'Curation::Page metainspector error'
+    end
+
+    def metatags
+      @metatags ||= metainspector.meta_tag['name']
     rescue
-      puts "MetaInspector error"
+      puts 'Curation::Page metatags error'
     end
   end
 end

data/lib/curation/version.rb

@@ -1,3 +1,3 @@
 module Curation
-  VERSION = "1.2"
+  VERSION = "1.7"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: curation
 version: !ruby/object:Gem::Version
-  version: '1.2'
+  version: '1.7'
 platform: ruby
 authors:
 - Arnaud Levy
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2020-07-20 00:00:00.000000000 Z
+date: 2020-07-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: metainspector