curation 1.4 → 1.8

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 71f7502a62d7bbb12799e869c1e5c64bd3e514c7ed97190790d42675451d48d1
-  data.tar.gz: 5cec6d6556dce75884b2e1a5e92e0d34744b623fcfb81b8b52684974a980f8b8
+  metadata.gz: b8086e74a861e147bce3a73dcec3cebef7d5d98ede440c911f57757feaaac354
+  data.tar.gz: 3d6ff271bcfbf8599a76653d60b2c26ed3832c1c174885d114912f95a7707ee7
 SHA512:
-  metadata.gz: 39dabe191b310e192e663836e00b8e4c28c716685cf05e0d86c807cc968b43d525480c8488b1d93e68c30b931d2d30560e445d3693458f78f937c76e38d788eb
-  data.tar.gz: ac5786df34fca74a9b7e6f336b630ce89baebf1720c01e48ac87c3f933457259276035587dbe0d922fa3096acb55cdfb392a1869b8c412eebaba214f66cd4f63
+  metadata.gz: '084c4c3fbdf3491530cd4ca6ca0fdc736df69a22a49cce40b0d7c8d169f6a45e7c5f01f280957016e32bedca93d593bb2d9e60141e148ee4f110935877543168'
+  data.tar.gz: 3fe95dde59a8cb2268a1f93ff1e8b7ccbca9212a7b0cdc0093c16925549545df422945db25ab29f250fcc05242b5d7b01eb822f3c97b4e2b96d19653d70dff39

data/.gitignore

@@ -6,3 +6,4 @@
 /pkg/
 /spec/reports/
 /tmp/
+.byebug_history

data/Gemfile

@@ -3,4 +3,7 @@ source "https://rubygems.org"
 # Specify your gem's dependencies in curation.gemspec
 gemspec
 
-gem "rake", "~> 12.0"
+gem 'rake', '~> 12.0'
+gem 'minitest'
+gem 'minitest-reporters'
+gem 'byebug'

data/Gemfile.lock

@@ -1,57 +1,94 @@
 PATH
   remote: .
   specs:
-    curation (1.4)
+    curation (1.8)
       metainspector
       nokogiri
 
 GEM
   remote: https://rubygems.org/
   specs:
-    addressable (2.7.0)
+    addressable (2.8.0)
       public_suffix (>= 2.0.2, < 5.0)
+    ansi (1.5.0)
+    builder (3.2.4)
+    byebug (11.1.3)
     domain_name (0.5.20190701)
       unf (>= 0.0.5, < 1.0.0)
-    faraday (1.0.1)
-      multipart-post (>= 1.2, < 3)
-    faraday-cookie_jar (0.0.6)
-      faraday (>= 0.7.4)
+    faraday (1.9.3)
+      faraday-em_http (~> 1.0)
+      faraday-em_synchrony (~> 1.0)
+      faraday-excon (~> 1.1)
+      faraday-httpclient (~> 1.0)
+      faraday-multipart (~> 1.0)
+      faraday-net_http (~> 1.0)
+      faraday-net_http_persistent (~> 1.0)
+      faraday-patron (~> 1.0)
+      faraday-rack (~> 1.0)
+      faraday-retry (~> 1.0)
+      ruby2_keywords (>= 0.0.4)
+    faraday-cookie_jar (0.0.7)
+      faraday (>= 0.8.0)
       http-cookie (~> 1.0.0)
+    faraday-em_http (1.0.0)
+    faraday-em_synchrony (1.0.0)
     faraday-encoding (0.0.5)
       faraday
+    faraday-excon (1.1.0)
     faraday-http-cache (2.2.0)
       faraday (>= 0.8)
-    faraday_middleware (1.0.0)
+    faraday-httpclient (1.0.1)
+    faraday-multipart (1.0.3)
+      multipart-post (>= 1.2, < 3)
+    faraday-net_http (1.0.1)
+    faraday-net_http_persistent (1.2.0)
+    faraday-patron (1.0.0)
+    faraday-rack (1.0.0)
+    faraday-retry (1.0.3)
+    faraday_middleware (1.2.0)
       faraday (~> 1.0)
-    fastimage (2.1.7)
-    http-cookie (1.0.3)
+    fastimage (2.2.6)
+    http-cookie (1.0.4)
       domain_name (~> 0.5)
-    metainspector (5.10.1)
-      addressable (~> 2.7.0)
-      faraday (~> 1.0.0)
-      faraday-cookie_jar (~> 0.0.6)
-      faraday-encoding (~> 0.0.5)
-      faraday-http-cache (~> 2.2.0)
-      faraday_middleware (~> 1.0.0)
-      fastimage (~> 2.1.7)
-      nesty (~> 1.0.2)
-      nokogiri (~> 1.10.9)
-    mini_portile2 (2.4.0)
+    metainspector (5.11.2)
+      addressable (~> 2.7)
+      faraday (~> 1.4)
+      faraday-cookie_jar (~> 0.0)
+      faraday-encoding (~> 0.0)
+      faraday-http-cache (~> 2.2)
+      faraday_middleware (~> 1.0)
+      fastimage (~> 2.2)
+      nesty (~> 1.0)
+      nokogiri (~> 1.11)
+    mini_portile2 (2.7.1)
+    minitest (5.15.0)
+    minitest-reporters (1.5.0)
+      ansi
+      builder
+      minitest (>= 5.0)
+      ruby-progressbar
     multipart-post (2.1.1)
     nesty (1.0.2)
-    nokogiri (1.10.10)
-      mini_portile2 (~> 2.4.0)
-    public_suffix (4.0.5)
+    nokogiri (1.13.1)
+      mini_portile2 (~> 2.7.0)
+      racc (~> 1.4)
+    public_suffix (4.0.6)
+    racc (1.6.0)
     rake (12.3.3)
+    ruby-progressbar (1.11.0)
+    ruby2_keywords (0.0.5)
     unf (0.1.4)
       unf_ext
-    unf_ext (0.0.7.7)
+    unf_ext (0.0.8)
 
 PLATFORMS
   ruby
 
 DEPENDENCIES
+  byebug
   curation!
+  minitest
+  minitest-reporters
   rake (~> 12.0)
 
 BUNDLED WITH

data/Rakefile

@@ -1,2 +1,10 @@
 require "bundler/gem_tasks"
-task :default => :spec
+require "rake/testtask"
+
+Rake::TestTask.new do |t|
+  t.libs << "test"
+  t.test_files = FileList["test/**/*_spec.rb"]
+  t.warning = false
+end
+
+task default: :test

data/lib/curation/version.rb

@@ -1,3 +1,3 @@
 module Curation
-  VERSION = "1.4"
+  VERSION = "1.8"
 end

data/lib/curation.rb

@@ -6,7 +6,7 @@ module Curation
   class Error < StandardError; end
 
   class Page
-    attr_reader :url, :title, :text, :image
+    attr_reader :url
 
     BLACKLIST = [
       'head', 'script', 'style', 'iframe', 'nav', 'noscript', 'header', 'footer', 'aside',
@@ -20,37 +20,28 @@ module Curation
     ]
 
     def initialize(url, html = nil)
-      @url = url
+      @url = url.to_s.gsub('http://', 'https://')
       @html = html
     end
 
     def title
-      @title = find_title
+      @title ||= find_title
     end
 
     def image
-      @image = find_image
-      @image = @image.to_s.gsub('http://', 'https://')
+      unless @image
+        @image = find_image
+        @image = @image.to_s.gsub('http://', 'https://')
+      end
       @image
     end
 
     def text
-      if json_ld.any?
-        json_ld.each do |ld|
-          next unless ld['@type'] == 'NewsArticle'
-          return ld['text'] if ld.has_key? 'text'
-          return ld['articleBody'] if ld.has_key? 'articleBody'
-        end
-      end
-      h = nokogiri.dup
-      BLACKLIST.each do |tag|
-        h.css(tag).remove
-      end
-      nodes = h.css('p')
-      nodes.xpath('//style').remove
-      text = nodes.to_html
-      text.gsub!('<br><br>', '<br>')
-      text
+      @text ||= find_text
+    end
+
+    def date
+      @date ||= find_date
     end
 
     protected
@@ -61,13 +52,17 @@ module Curation
           return ld['headline'] if ld.has_key? 'headline'
         end
       end
-      [
-        metainspector.best_title,
-        metainspector.title,
-        nokogiri.css('[itemprop="headline"]')&.first&.inner_text,
-        nokogiri.css('title')&.first&.inner_text
-      ].each do |possibility|
-        return possibility unless possibility.blank?
+      begin
+        [
+          metainspector.best_title,
+          metainspector.title,
+          nokogiri.css('[itemprop="headline"]')&.first&.inner_text,
+          nokogiri.css('title')&.first&.inner_text
+        ].each do |possibility|
+          return possibility unless possibility.to_s.empty?
+        end
+      rescue
+        puts 'Curation::Page find_title error'
       end
       return ''
     end
@@ -87,48 +82,140 @@ module Curation
           end
         end
       end
-      [
-        metainspector.images.best,
-        nokogiri.css('[property="og:image"]').first&.attributes["content"].value
-      ].each do |possibility|
-        return possibility unless possibility.blank?
+      begin
+        [
+          metainspector.images.best,
+          nokogiri.css('[property="og:image"]').first&.attributes['content'].value
+        ].each do |possibility|
+          return possibility unless possibility.to_s.empty?
+        end
+      rescue
+        puts 'Curation::Page find_image error'
       end
       return ''
     end
 
+    def find_text
+      if json_ld.any?
+        json_ld.each do |ld|
+          next unless ['NewsArticle', 'ReportageNewsArticle'].include? ld['@type']
+          return ld['text'] if ld.has_key? 'text'
+          return ld['articleBody'] if ld.has_key? 'articleBody'
+        end
+      end
+      h = nokogiri.dup
+      BLACKLIST.each do |tag|
+        h.css(tag).remove
+      end
+      nodes = h.css('p')
+      nodes.xpath('//style').remove
+      text = nodes.to_html
+      text.gsub!('<br><br>', '<br>')
+      text
+    end
+
+    def find_date
+      if json_ld.any?
+        json_ld.each do |ld|
+          next unless ['NewsArticle', 'ReportageNewsArticle'].include? ld['@type']
+          return Date.parse ld['datePublished'] if ld.has_key? 'datePublished'
+        end
+      end
+      return Date.parse metatags['date'] rescue nil
+      return Date.parse metatags['pubdate'] rescue nil
+      return Date.parse nokogiri.css('meta[property="article:published"]').first['content'] rescue nil
+      return Date.parse nokogiri.css('meta[property="article:published_time"]').first['content'] rescue nil
+      chunks = html.split('DisplayDate')
+      if chunks.count > 1
+        value = chunks[1]
+        value = value.split(',').first
+        value = value.gsub('"', '')
+        value = value[1..-1] if value[0] == ':'
+        return Date.parse value rescue nil
+      end
+      begin
+        value = nokogiri.css('.postDate').first
+        value = value.inner_text
+        value = value.gsub(' — ', '')
+        return Date.parse value
+      rescue
+      end
+      begin
+        value = nokogiri.css('.gta_post_date').first
+        value = value.inner_text
+        return Date.parse value
+      rescue
+      end
+    end
+
+    private
+
     def json_ld
-      unless @json_ld
+      unless defined?(@json_ld)
         @json_ld = []
         begin
           options = nokogiri.css('[type="application/ld+json"]')
           options.each do |option|
+            # require 'byebug'; byebug
             string = option.inner_text
             hash = JSON.parse(string)
             @json_ld << hash
           end
+          # Some sites have tables in tables
+          @json_ld.flatten!
         rescue
-          puts "JSON LD error"
+          puts 'Curation::Page json_ld error'
         end
       end
       @json_ld
     end
 
+    def file
+      @file ||= URI.open url, 'User-Agent' => "Mozilla/5.0"
+    rescue
+      puts "Curation::Page file error with url #{url}"
+    end
+
     def html
-      @html ||= URI.open url
+      unless @html
+        file.rewind
+        @html = file.read
+        file.rewind
+      end
+      @html
     rescue
-      puts "Impossible to open #{url}"
+      puts "Curation::Page html error"
     end
 
     def nokogiri
-      @nokogiri ||= Nokogiri::HTML html
+      unless @nokogiri
+        if file.nil?
+          @nokogiri = metainspector.parsed
+        else
+          file.rewind
+          @nokogiri = Nokogiri::HTML file
+          file.rewind
+        end
+      end
+      @nokogiri
     rescue
-      puts "Nokogiri error"
+      puts 'Curation::Page nokogiri error'
     end
 
     def metainspector
-      @metainspector ||= MetaInspector.new url, document: html
+      unless @metainspector
+        @metainspector = html.nil?  ? MetaInspector.new(url)
+                                    : MetaInspector.new(url, document: html)
+      end
+      @metainspector
+    rescue
+      puts 'Curation::Page metainspector error'
+    end
+
+    def metatags
+      @metatags ||= metainspector.meta_tag['name']
     rescue
-      puts "MetaInspector error"
+      puts 'Curation::Page metatags error'
     end
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: curation
 version: !ruby/object:Gem::Version
-  version: '1.4'
+  version: '1.8'
 platform: ruby
 authors:
 - Arnaud Levy
-autorequire: 
+autorequire:
 bindir: exe
 cert_chain: []
-date: 2020-07-22 00:00:00.000000000 Z
+date: 2022-02-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: metainspector
@@ -65,7 +65,7 @@ licenses:
 metadata:
   homepage_uri: https://github.com/arnaudlevy/curation
   source_code_uri: https://github.com/arnaudlevy/curation
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -80,8 +80,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.3
-signing_key: 
+rubygems_version: 3.1.6
+signing_key:
 specification_version: 4
 summary: Curation of content
 test_files: []