curation 1.11 → 2.0.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: bcbcb0a8ecb81b659e2ae2aa35e27acac6006265cf2743b6394f81203c634425
-  data.tar.gz: 25b25c7f30be8f9b004cec7efbd41caae4e69e87000ff585b92c8377b7855439
+  metadata.gz: c2062c7ec7fb444d27f102d26658b386a01a964ea4e03aa0d81a472316012d11
+  data.tar.gz: 634f5216e61801b3ac42c8b340d5012c101024f4d5b5e0ba155c01a89f38cafb
 SHA512:
-  metadata.gz: 420056127e2c0ca86a4ad2e08fb37aa3da31a394264e4d7807a844dd9db61bc44124d6e978a076d9f50a1df37f7a5f80cc782145f54d11c7c3ce00566feca80d
-  data.tar.gz: a3b387e1c7345968c0eaaf1d2e8fbdbf752aa8b7a8b1e3b3807f206c43e7b096419b61b907622065f690d00a55b933038f0f3ed1b4fefa4d970f272afc1438d5
+  metadata.gz: 15008b92c6a51fdf9bd79b9f1da01d59323c13589b6e8ccd5d8754b96153173fe8a2cdffa5faa45a4d38c92b097a35012f8dff98fea6244ade8c9af22c13d1cc
+  data.tar.gz: 96a29d3c8482fce0101f91a3f24eafaec51a9115f2169831b67939edf79a47d7f5bb0c3223466de355cf8a367143a262cc54a1d5e4dd7e6bffe2d310879a9cae

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    curation (1.11)
+    curation (2.0.1)
       htmlentities
       metainspector
       nokogiri
@@ -12,11 +12,13 @@ GEM
     addressable (2.8.5)
       public_suffix (>= 2.0.2, < 6.0)
     ansi (1.5.0)
+    base64 (0.2.0)
     builder (3.2.4)
     byebug (11.1.3)
     domain_name (0.5.20190701)
       unf (>= 0.0.5, < 1.0.0)
-    faraday (2.7.10)
+    faraday (2.7.11)
+      base64
       faraday-net_http (>= 2.0, < 3.1)
       ruby2_keywords (>= 0.0.4)
     faraday-cookie_jar (0.0.7)
@@ -50,7 +52,7 @@ GEM
       fastimage (~> 2.2)
       nesty (~> 1.0)
       nokogiri (~> 1.13)
-    minitest (5.19.0)
+    minitest (5.20.0)
     minitest-reporters (1.6.1)
       ansi
       builder
@@ -60,13 +62,13 @@ GEM
     nokogiri (1.15.4-x86_64-darwin)
       racc (~> 1.4)
     public_suffix (5.0.3)
-    racc (1.7.1)
+    racc (1.7.3)
     rake (12.3.3)
     ruby-progressbar (1.13.0)
     ruby2_keywords (0.0.5)
     unf (0.1.4)
       unf_ext
-    unf_ext (0.0.8.2)
+    unf_ext (0.0.9)
     zlib (2.1.1)
 
 PLATFORMS

data/LICENSE.txt

@@ -1,6 +1,6 @@
 The MIT License (MIT)
 
-Copyright (c) 2020 Arnaud Levy
+Copyright (c) 2020 noesya
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

data/README.md

@@ -37,7 +37,7 @@ To install this gem onto your local machine, run `bundle exec rake install`. To
 
 ## Contributing
 
-Bug reports and pull requests are welcome on GitHub at https://github.com/[USERNAME]/curation. This project is intended to be a safe, welcoming space for collaboration, and contributors are expected to adhere to the [code of conduct](https://github.com/[USERNAME]/curation/blob/master/CODE_OF_CONDUCT.md).
+Bug reports and pull requests are welcome on GitHub at https://github.com/noesya/curation. This project is intended to be a safe, welcoming space for collaboration, and contributors are expected to adhere to the [code of conduct](https://github.com/noesya/curation/blob/master/CODE_OF_CONDUCT.md).
 
 
 ## License

data/curation.gemspec

@@ -8,12 +8,12 @@ Gem::Specification.new do |spec|
 
   spec.summary       = 'Curation of content'
   spec.description   = %q{When you build content curation tools, you need to extract the content of pages (title, text, image...). This requires different strategies and some fine tuning to work efficiently.}
-  spec.homepage      = "https://github.com/arnaudlevy/curation"
+  spec.homepage      = "https://github.com/noesya/curation"
   spec.license       = "MIT"
   spec.required_ruby_version = Gem::Requirement.new(">= 2.3.0")
 
   spec.metadata["homepage_uri"] = spec.homepage
-  spec.metadata["source_code_uri"] = "https://github.com/arnaudlevy/curation"
+  spec.metadata["source_code_uri"] = "https://github.com/noesya/curation"
 
   spec.files         = Dir.chdir(File.expand_path('..', __FILE__)) do
     `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }

data/lib/curation/finders/image.rb

@@ -0,0 +1,51 @@
+module Image
+
+  def image
+    @image ||= find_image.to_s.gsub('http://', 'https://')
+  end
+
+  protected
+
+  def find_image
+    log "Curation::Page find_image #{url}"
+    if json_ld.any?
+      json_ld.each do |ld|
+        ld = ld.first if ld.is_a?(Array)
+        if ld.has_key? 'image'
+          image_data = ld['image']
+          if image_data.is_a? String
+            log "Curation::Page find_image json_ld string"
+            return image_data 
+          end
+          if image_data.is_a? Array
+            first = image_data.first
+            if first.is_a? String
+              log "Curation::Page find_image json_ld array"
+              return first 
+            end
+            if first.is_a? Hash
+              log "Curation::Page find_image json_ld array url"
+              return first['url'] 
+            end
+          end
+          if image_data.is_a? Hash
+            log "Curation::Page find_image json_ld url"
+            return image_data['url'] 
+          end
+        end
+      end
+    end
+    begin
+      [
+        metainspector.images.best,
+        nokogiri.css('[property="og:image"]').first&.attributes['content'].value
+      ].each do |possibility|
+        return possibility unless possibility.to_s.empty?
+      end
+    rescue
+      puts 'Curation::Page find_image error'
+    end
+    return ''
+  end
+
+end

data/lib/curation/finders/publication_date.rb

@@ -0,0 +1,43 @@
+module PublicationDate
+
+  def date
+    @date ||= find_date
+  end
+
+  protected
+
+  def find_date
+    if json_ld.any?
+      json_ld.each do |ld|
+        next unless ['NewsArticle', 'ReportageNewsArticle'].include? ld['@type']
+        return Date.parse ld['datePublished'] if ld.has_key? 'datePublished'
+      end
+    end
+    return Date.parse metatags['date'] rescue nil
+    return Date.parse metatags['pubdate'] rescue nil
+    return Date.parse nokogiri.css('meta[property="article:published"]').first['content'] rescue nil
+    return Date.parse nokogiri.css('meta[property="article:published_time"]').first['content'] rescue nil
+    return Date.parse nokogiri.css('meta[property="og:article:published_time"]').first['content'] rescue nil
+    chunks = html.split('DisplayDate')
+    if chunks.count > 1
+      value = chunks[1]
+      value = value.split(',').first
+      value = value.gsub('"', '')
+      value = value[1..-1] if value[0] == ':'
+      return Date.parse value rescue nil
+    end
+    begin
+      value = nokogiri.css('.postDate').first
+      value = value.inner_text
+      value = value.gsub(' — ', '')
+      return Date.parse value
+    rescue
+    end
+    begin
+      value = nokogiri.css('.gta_post_date').first
+      value = value.inner_text
+      return Date.parse value
+    rescue
+    end
+  end
+end

data/lib/curation/finders/text.rb

@@ -0,0 +1,90 @@
+module Text
+
+  def text
+    @text ||= find_text_and_clean
+  end
+
+  protected
+
+  BLACKLIST_HARD = [
+    'head', 'script', 'style', 'iframe', 'nav', 'noscript', 'header', 'footer', 'aside',
+    '.navigation', '.top-menu-container', '.navbar', '.navbar-header', '.breadcrumb',
+    '#breadcrumbs', '[typeof="v:Breadcrumb"]', '.skip-link', '.search', '.search-form',
+    '.categories', '.post-categories', '.datas', '.post-datas', '.twitter-media',
+    '.instagram-media', '.widget', '.related-post-tags', '.social-list', '.top-scroll',
+    '.comments', '.signature', '.publicite', '.footer', '.Footer', '.footer-copyright',
+    '[itemprop*="author"]', '[style*="display:none;"]', '[style*="display:none"]',
+    '[style*="display: none;"]', '[style*="display: none"]', '[aria-hidden="true"]'
+  ]
+
+  BLACKLIST_SOFT = [
+    'head', 'script', 'noscript', 'style', 'iframe', 'nav', 'footer', 'aside', '[role="dialog"]'
+  ]
+
+  def find_text_and_clean
+    text = find_text.to_s.dup
+    text = text.gsub('<br><br>', '<br>')
+    text = text.gsub(/\s+/, ' ')
+    text = clean_encoding(text)
+    text
+
+  end
+
+  def find_text
+    find_text_with_json_ld || 
+    find_text_with_nokogiri_hard ||
+    find_text_with_nokogiri_soft
+  end
+
+  def find_text_with_json_ld
+    if json_ld.any?
+      json_ld.each do |ld|
+        next unless ['NewsArticle', 'ReportageNewsArticle'].include? ld['@type']
+        return ld['text'] if ld.has_key? 'text'
+        return ld['articleBody'] if ld.has_key? 'articleBody'
+      end
+    end
+    false
+  end
+
+  def find_text_with_nokogiri_hard
+    h = nokogiri.dup
+    h.xpath('//style').remove
+    BLACKLIST_HARD.each do |tag|
+      h.css(tag).remove
+    end
+    nodes = h.css('p')
+    text = nodes.to_html
+    text.present? ? text : false
+  end
+
+  def find_text_with_nokogiri_soft
+    h = nokogiri.dup
+    h.xpath('//style').remove
+    BLACKLIST_SOFT.each do |tag|
+      h.css(tag).remove
+    end
+    h.text
+  end
+
+  # r&Atilde;&copy;forme -> réforme
+  def clean_encoding(text)
+    clean_text = HTMLEntities.new.decode text
+    double_encoding = false
+    [
+      'é', # é
+      'è', # è
+      'î', # î
+      'ê', # ê
+    ].each do |string|
+      double_encoding = true if clean_text.include? string
+    end
+    if double_encoding
+      clean_text.encode('iso-8859-1', undef: :replace)
+                .force_encoding('utf-8')
+    else
+      text
+    end
+  end
+
+end

data/lib/curation/finders/title.rb

@@ -0,0 +1,58 @@
+module Title
+
+  def title
+    @title ||= find_title.strip.gsub(/\s+/, ' ')
+  end
+
+  protected
+
+  def find_title
+    find_title_with_json_ld ||
+    find_title_with_metainspector ||
+    find_title_with_nokogiri ||
+    ''
+  end
+
+  def find_title_with_json_ld
+    if json_ld.any?
+      json_ld.each do |ld|
+        # require 'byebug'; byebug
+        ld = ld.first if ld.is_a?(Array)
+        return ld['headline'] if ld.has_key? 'headline'
+      end
+    end
+    false
+  end
+
+  def find_title_with_metainspector
+    metainspector_best_title = metainspector.best_title
+    metainspector_title = metainspector.title
+    # Problème avec une balise <meta property="title" content="Run 0"  />,
+    # metainspector croit que c'est le titre de la page.
+    # Comme le title contient le best title, avec souvent des infos en plus sur le site, 
+    # on vérifie si le best title est bien contenu dans le title
+    if  metainspector_title.present? && 
+        metainspector_title.present? &&
+        metainspector_best_title.present? && 
+        metainspector_title.include?(metainspector_best_title)
+      return metainspector_best_title
+    elsif metainspector_title.present?
+      return metainspector_title
+    end
+    false
+  end
+
+  def find_title_with_nokogiri
+    begin
+      [
+        nokogiri.css('[itemprop="headline"]')&.first&.inner_text,
+        nokogiri.css('title')&.first&.inner_text
+      ].each do |possibility|
+        return possibility unless possibility.to_s.empty?
+      end
+    rescue
+      log 'Curation::Page find_title_with_nokogiri error'
+    end
+  end
+
+end

data/lib/curation/tools/jsonld.rb

@@ -0,0 +1,26 @@
+module Jsonld
+
+  def json_ld
+    unless defined?(@json_ld)
+      @json_ld = []
+      begin
+        options = nokogiri.css('[type="application/ld+json"]')
+        options.each do |option|
+          @json_ld << json_ld_from_object(option)
+        end
+        # Some sites have tables in tables
+        @json_ld.flatten!
+        # require 'byebug'; byebug
+      rescue
+        log 'Curation::Page json_ld error'
+      end
+    end
+    @json_ld
+  end
+
+  def json_ld_from_object(object)
+    JSON.parse object.inner_text
+  rescue
+    {}
+  end
+end

data/lib/curation/tools/metainspector.rb

@@ -0,0 +1,18 @@
+module Metainspector
+
+  def metainspector
+    unless @metainspector
+      @metainspector = html.nil?  ? MetaInspector.new(url)
+                                  : MetaInspector.new(url, document: html)
+    end
+    @metainspector
+  rescue
+    log 'Curation::Page metainspector error'
+  end
+
+  def metatags
+    @metatags ||= metainspector.meta_tag['name']
+  rescue
+    log 'Curation::Page metatags error'
+  end
+end

data/lib/curation/tools/nokogiri.rb

@@ -0,0 +1,17 @@
+module Nokogiri
+
+  def nokogiri
+    unless @nokogiri
+      if file.nil?
+        @nokogiri = metainspector.parsed
+      else
+        file.rewind
+        @nokogiri = Nokogiri::HTML file
+        file.rewind
+      end
+    end
+    @nokogiri
+  rescue
+    log 'Curation::Page nokogiri error'
+  end
+end

data/lib/curation/tools/raw.rb

@@ -0,0 +1,19 @@
+module Raw
+
+  def file
+    @file ||= URI.open url, 'User-Agent' => "Mozilla/5.0"
+  rescue
+    log "Curation::Page file error with url #{url}"
+  end
+
+  def html
+    unless @html
+      file.rewind
+      @html = file.read
+      file.rewind
+    end
+    @html
+  rescue
+    log "Curation::Page html error"
+  end
+end

data/lib/curation/version.rb

@@ -1,3 +1,3 @@
 module Curation
-  VERSION = "1.11"
+  VERSION = "2.0.1"
 end

data/lib/curation.rb

@@ -1,4 +1,12 @@
 require "curation/version"
+require "curation/tools/raw"
+require "curation/tools/nokogiri"
+require "curation/tools/jsonld"
+require "curation/tools/metainspector"
+require "curation/finders/image"
+require "curation/finders/publication_date"
+require "curation/finders/text"
+require "curation/finders/title"
 require "metainspector"
 require "open-uri"
 require "htmlentities"
@@ -7,274 +15,29 @@ module Curation
   class Error < StandardError; end
 
   class Page
+    # Tools
+    include Raw
+    include Jsonld
+    include Metainspector
+    include Nokogiri
+
+    # Finders
+    include Title
+    include Image
+    include PublicationDate
+    include Text
+
     attr_reader :url
     attr_accessor :verbose
 
-    BLACKLIST = [
-      'head', 'script', 'style', 'iframe', 'nav', 'noscript', 'header', 'footer', 'aside',
-      '.navigation', '.top-menu-container', '.navbar', '.navbar-header', '.breadcrumb',
-      '#breadcrumbs', '[typeof="v:Breadcrumb"]', '.skip-link', '.search', '.search-form',
-      '.categories', '.post-categories', '.datas', '.post-datas', '.twitter-media',
-      '.instagram-media', '.widget', '.related-post-tags', '.social-list', '.top-scroll',
-      '.comments', '.signature', '.publicite', '.footer', '.Footer', '.footer-copyright',
-      '[itemprop*="author"]', '[style*="display:none;"]', '[style*="display:none"]',
-      '[style*="display: none;"]', '[style*="display: none"]', '[aria-hidden="true"]'
-    ]
-
     def initialize(url, html = nil)
       @url = url.to_s.gsub('http://', 'https://')
       @html = html
       @verbose = false
     end
 
-    def title
-      @title ||= find_title
-    end
-
-    def image
-      unless @image
-        @image = find_image
-        @image = @image.to_s.gsub('http://', 'https://')
-      end
-      @image
-    end
-
-    def text
-      # require 'byebug'; byebug
-      @text ||= find_text
-    end
-
-    def date
-      @date ||= find_date
-    end
-
     protected
 
-    def find_title
-      if json_ld.any?
-        json_ld.each do |ld|
-          # require 'byebug'; byebug
-          ld = ld.first if ld.is_a?(Array)
-          return ld['headline'] if ld.has_key? 'headline'
-        end
-      end
-      begin
-        [
-          metainspector.best_title,
-          metainspector.title,
-          nokogiri.css('[itemprop="headline"]')&.first&.inner_text,
-          nokogiri.css('title')&.first&.inner_text
-        ].each do |possibility|
-          return possibility unless possibility.to_s.empty?
-        end
-      rescue
-        log 'Curation::Page find_title error'
-      end
-      return ''
-    end
-
-    def find_image
-      log "Curation::Page find_image #{url}"
-      if json_ld.any?
-        json_ld.each do |ld|
-          ld = ld.first if ld.is_a?(Array)
-          if ld.has_key? 'image'
-            image_data = ld['image']
-            if image_data.is_a? String
-              log "Curation::Page find_image json_ld string"
-              return image_data 
-            end
-            if image_data.is_a? Array
-              first = image_data.first
-              if first.is_a? String
-                log "Curation::Page find_image json_ld array"
-                return first 
-              end
-              if first.is_a? Hash
-                log "Curation::Page find_image json_ld array url"
-                return first['url'] 
-              end
-            end
-            if image_data.is_a? Hash
-              log "Curation::Page find_image json_ld url"
-              return image_data['url'] 
-            end
-          end
-        end
-      end
-      begin
-        [
-          metainspector.images.best,
-          nokogiri.css('[property="og:image"]').first&.attributes['content'].value
-        ].each do |possibility|
-          return possibility unless possibility.to_s.empty?
-        end
-      rescue
-        puts 'Curation::Page find_image error'
-      end
-      return ''
-    end
-
-    def find_text
-      text = find_text_with_json_ld || find_text_with_nokogiri
-      text.to_s.gsub!('<br><br>', '<br>')
-      # require 'byebug'; byebug
-      text = clean_encoding text
-      text
-    end
-
-    def find_text_with_json_ld
-      if json_ld.any?
-        json_ld.each do |ld|
-          next unless ['NewsArticle', 'ReportageNewsArticle'].include? ld['@type']
-          return ld['text'] if ld.has_key? 'text'
-          return ld['articleBody'] if ld.has_key? 'articleBody'
-        end
-      end
-      nil
-    end
-
-    def find_text_with_nokogiri
-      h = nokogiri.dup
-      BLACKLIST.each do |tag|
-        h.css(tag).remove
-      end
-      nodes = h.css('p')
-      nodes.xpath('//style').remove
-      text = nodes.to_html
-      text
-    end
-
-    def find_date
-      if json_ld.any?
-        json_ld.each do |ld|
-          next unless ['NewsArticle', 'ReportageNewsArticle'].include? ld['@type']
-          return Date.parse ld['datePublished'] if ld.has_key? 'datePublished'
-        end
-      end
-      return Date.parse metatags['date'] rescue nil
-      return Date.parse metatags['pubdate'] rescue nil
-      return Date.parse nokogiri.css('meta[property="article:published"]').first['content'] rescue nil
-      return Date.parse nokogiri.css('meta[property="article:published_time"]').first['content'] rescue nil
-      return Date.parse nokogiri.css('meta[property="og:article:published_time"]').first['content'] rescue nil
-      chunks = html.split('DisplayDate')
-      if chunks.count > 1
-        value = chunks[1]
-        value = value.split(',').first
-        value = value.gsub('"', '')
-        value = value[1..-1] if value[0] == ':'
-        return Date.parse value rescue nil
-      end
-      begin
-        value = nokogiri.css('.postDate').first
-        value = value.inner_text
-        value = value.gsub(' — ', '')
-        return Date.parse value
-      rescue
-      end
-      begin
-        value = nokogiri.css('.gta_post_date').first
-        value = value.inner_text
-        return Date.parse value
-      rescue
-      end
-    end
-
-    private
-
-    def json_ld
-      unless defined?(@json_ld)
-        @json_ld = []
-        begin
-          options = nokogiri.css('[type="application/ld+json"]')
-          options.each do |option|
-            @json_ld << json_ld_from_object(option)
-          end
-          # Some sites have tables in tables
-          @json_ld.flatten!
-          # require 'byebug'; byebug
-        rescue
-          log 'Curation::Page json_ld error'
-        end
-      end
-      @json_ld
-    end
-
-    def json_ld_from_object(object)
-      JSON.parse object.inner_text
-    rescue
-      {}
-    end
-
-    def file
-      @file ||= URI.open url, 'User-Agent' => "Mozilla/5.0"
-    rescue
-      log "Curation::Page file error with url #{url}"
-    end
-
-    def html
-      unless @html
-        file.rewind
-        @html = file.read
-        file.rewind
-      end
-      @html
-    rescue
-      log "Curation::Page html error"
-    end
-
-    def nokogiri
-      unless @nokogiri
-        if file.nil?
-          @nokogiri = metainspector.parsed
-        else
-          file.rewind
-          @nokogiri = Nokogiri::HTML file
-          file.rewind
-        end
-      end
-      @nokogiri
-    rescue
-      log 'Curation::Page nokogiri error'
-    end
-
-    def metainspector
-      unless @metainspector
-        @metainspector = html.nil?  ? MetaInspector.new(url)
-                                    : MetaInspector.new(url, document: html)
-      end
-      @metainspector
-    rescue
-      log 'Curation::Page metainspector error'
-    end
-
-    def metatags
-      @metatags ||= metainspector.meta_tag['name']
-    rescue
-      log 'Curation::Page metatags error'
-    end
-
-    # r&Atilde;&copy;forme -> réforme
-    def clean_encoding(text)
-      clean_text = HTMLEntities.new.decode text
-      double_encoding = false
-      [
-        'é', # é
-        'è', # è
-        'î', # î
-        'ê', # ê
-      ].each do |string|
-        # require 'byebug'; byebug
-        double_encoding = true if clean_text.include? string
-      end
-      if double_encoding
-        clean_text.encode('iso-8859-1', undef: :replace)
-                  .force_encoding('utf-8')
-      else
-        text
-      end
-    end
-
     def log(message)
       puts message if verbose
     end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: curation
 version: !ruby/object:Gem::Version
-  version: '1.11'
+  version: 2.0.1
 platform: ruby
 authors:
 - Arnaud Levy
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2023-08-29 00:00:00.000000000 Z
+date: 2023-11-11 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: metainspector
@@ -72,13 +72,21 @@ files:
 - bin/setup
 - curation.gemspec
 - lib/curation.rb
+- lib/curation/finders/image.rb
+- lib/curation/finders/publication_date.rb
+- lib/curation/finders/text.rb
+- lib/curation/finders/title.rb
+- lib/curation/tools/jsonld.rb
+- lib/curation/tools/metainspector.rb
+- lib/curation/tools/nokogiri.rb
+- lib/curation/tools/raw.rb
 - lib/curation/version.rb
-homepage: https://github.com/arnaudlevy/curation
+homepage: https://github.com/noesya/curation
 licenses:
 - MIT
 metadata:
-  homepage_uri: https://github.com/arnaudlevy/curation
-  source_code_uri: https://github.com/arnaudlevy/curation
+  homepage_uri: https://github.com/noesya/curation
+  source_code_uri: https://github.com/noesya/curation
 post_install_message:
 rdoc_options: []
 require_paths:
@@ -94,7 +102,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.4.6
+rubygems_version: 3.4.10
 signing_key:
 specification_version: 4
 summary: Curation of content