curation 1.10 → 2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: a23e967e5d017ce61f9719647f45c20e1336aff067bc2323f64641dfac695f75
-  data.tar.gz: 49427d9325a27034c1969d71875dcd4eacff4f5c3ac9625ccc6cdb554c4c2df4
+  metadata.gz: 8bfaa5cfe719f5a5f5c9f153a358e74f11eb4217f4bbddca880b8f45829fc913
+  data.tar.gz: 4abb219144b696da0f33ab12b251036bd67c4ca7250a4576febdcaaa8e409dc0
 SHA512:
-  metadata.gz: b01f4209b09d6ec09917096159b98be71b31fd0952524a12c7310ba29f88ee8c888bc2927015ef2b1fb529cfe46964c08c743c818807518dee6bfa3cc32f6767
-  data.tar.gz: 564d14e3afaa17f00ac7b034917c7b28612b54b6f71126a36de4d6d43def7b8c19814475ee5b13651bbea7e7eed752b873f918af10baf169bd153d66b8d05c7d
+  metadata.gz: 6d19e2a775ea069a3e6b0dc50b66f30d52f0c4e1e101cea2c9f0dde5cedde611043cf4267a4764994be8105e422919bf708f850935d531d9bb8f907025393a05
+  data.tar.gz: 4f0dd67b073cc2003cf798603f7221ba32977645180b671bcde0b1464fc53085cf9a076c2f30f4ce3775a8524246386d576e1bd0d1021c8388db58d13c16068f

data/Gemfile.lock

@@ -1,88 +1,79 @@
 PATH
   remote: .
   specs:
-    curation (1.10)
+    curation (2.0)
       htmlentities
-      metainspector (~> 5.12)
+      metainspector
       nokogiri
 
 GEM
   remote: https://rubygems.org/
   specs:
-    addressable (2.8.0)
-      public_suffix (>= 2.0.2, < 5.0)
+    addressable (2.8.5)
+      public_suffix (>= 2.0.2, < 6.0)
     ansi (1.5.0)
+    base64 (0.2.0)
     builder (3.2.4)
     byebug (11.1.3)
     domain_name (0.5.20190701)
       unf (>= 0.0.5, < 1.0.0)
-    faraday (1.10.0)
-      faraday-em_http (~> 1.0)
-      faraday-em_synchrony (~> 1.0)
-      faraday-excon (~> 1.1)
-      faraday-httpclient (~> 1.0)
-      faraday-multipart (~> 1.0)
-      faraday-net_http (~> 1.0)
-      faraday-net_http_persistent (~> 1.0)
-      faraday-patron (~> 1.0)
-      faraday-rack (~> 1.0)
-      faraday-retry (~> 1.0)
+    faraday (2.7.11)
+      base64
+      faraday-net_http (>= 2.0, < 3.1)
       ruby2_keywords (>= 0.0.4)
     faraday-cookie_jar (0.0.7)
       faraday (>= 0.8.0)
       http-cookie (~> 1.0.0)
-    faraday-em_http (1.0.0)
-    faraday-em_synchrony (1.0.0)
     faraday-encoding (0.0.5)
       faraday
-    faraday-excon (1.1.0)
-    faraday-http-cache (2.4.0)
+    faraday-follow_redirects (0.3.0)
+      faraday (>= 1, < 3)
+    faraday-gzip (1.0.0)
+      faraday (>= 1.0)
+      zlib (~> 2.1)
+    faraday-http-cache (2.5.0)
       faraday (>= 0.8)
-    faraday-httpclient (1.0.1)
-    faraday-multipart (1.0.4)
-      multipart-post (~> 2)
-    faraday-net_http (1.0.1)
-    faraday-net_http_persistent (1.2.0)
-    faraday-patron (1.0.0)
-    faraday-rack (1.0.0)
-    faraday-retry (1.0.3)
-    faraday_middleware (1.2.0)
-      faraday (~> 1.0)
-    fastimage (2.2.6)
+    faraday-net_http (3.0.2)
+    faraday-retry (2.2.0)
+      faraday (~> 2.0)
+    fastimage (2.2.7)
     htmlentities (4.3.4)
     http-cookie (1.0.5)
       domain_name (~> 0.5)
-    metainspector (5.12.1)
-      addressable (~> 2.7)
-      faraday (>= 1.4, < 3.0)
+    metainspector (5.15.0)
+      addressable (~> 2.8.4)
+      faraday (~> 2.5)
       faraday-cookie_jar (~> 0.0)
       faraday-encoding (~> 0.0)
-      faraday-http-cache (~> 2.2)
-      faraday_middleware (~> 1.0)
+      faraday-follow_redirects (~> 0.3)
+      faraday-gzip (>= 0.1, < 2.0)
+      faraday-http-cache (~> 2.5)
+      faraday-retry (~> 2.0)
       fastimage (~> 2.2)
       nesty (~> 1.0)
-      nokogiri (~> 1.11)
-    minitest (5.15.0)
-    minitest-reporters (1.5.0)
+      nokogiri (~> 1.13)
+    minitest (5.20.0)
+    minitest-reporters (1.6.1)
       ansi
       builder
       minitest (>= 5.0)
       ruby-progressbar
-    multipart-post (2.2.0)
     nesty (1.0.2)
-    nokogiri (1.13.6-x86_64-darwin)
+    nokogiri (1.15.4-x86_64-darwin)
       racc (~> 1.4)
-    public_suffix (4.0.7)
-    racc (1.6.0)
+    public_suffix (5.0.3)
+    racc (1.7.3)
     rake (12.3.3)
-    ruby-progressbar (1.11.0)
+    ruby-progressbar (1.13.0)
     ruby2_keywords (0.0.5)
     unf (0.1.4)
       unf_ext
-    unf_ext (0.0.8.2)
+    unf_ext (0.0.9)
+    zlib (2.1.1)
 
 PLATFORMS
   x86_64-darwin-21
+  x86_64-darwin-22
 
 DEPENDENCIES
   byebug
@@ -92,4 +83,4 @@ DEPENDENCIES
   rake (~> 12.0)
 
 BUNDLED WITH
-   2.3.12
+   2.4.6

data/LICENSE.txt

@@ -1,6 +1,6 @@
 The MIT License (MIT)
 
-Copyright (c) 2020 Arnaud Levy
+Copyright (c) 2020 noesya
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

data/README.md

@@ -37,7 +37,7 @@ To install this gem onto your local machine, run `bundle exec rake install`. To
 
 ## Contributing
 
-Bug reports and pull requests are welcome on GitHub at https://github.com/[USERNAME]/curation. This project is intended to be a safe, welcoming space for collaboration, and contributors are expected to adhere to the [code of conduct](https://github.com/[USERNAME]/curation/blob/master/CODE_OF_CONDUCT.md).
+Bug reports and pull requests are welcome on GitHub at https://github.com/noesya/curation. This project is intended to be a safe, welcoming space for collaboration, and contributors are expected to adhere to the [code of conduct](https://github.com/noesya/curation/blob/master/CODE_OF_CONDUCT.md).
 
 
 ## License

data/curation.gemspec

@@ -8,12 +8,12 @@ Gem::Specification.new do |spec|
 
   spec.summary       = 'Curation of content'
   spec.description   = %q{When you build content curation tools, you need to extract the content of pages (title, text, image...). This requires different strategies and some fine tuning to work efficiently.}
-  spec.homepage      = "https://github.com/arnaudlevy/curation"
+  spec.homepage      = "https://github.com/noesya/curation"
   spec.license       = "MIT"
   spec.required_ruby_version = Gem::Requirement.new(">= 2.3.0")
 
   spec.metadata["homepage_uri"] = spec.homepage
-  spec.metadata["source_code_uri"] = "https://github.com/arnaudlevy/curation"
+  spec.metadata["source_code_uri"] = "https://github.com/noesya/curation"
 
   spec.files         = Dir.chdir(File.expand_path('..', __FILE__)) do
     `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
@@ -21,7 +21,7 @@ Gem::Specification.new do |spec|
   spec.bindir        = "exe"
   spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
   spec.require_paths = ["lib"]
-  spec.add_dependency "metainspector", '~> 5.12'
+  spec.add_dependency "metainspector"
   spec.add_dependency "nokogiri"
   spec.add_dependency "htmlentities"
 end

data/lib/curation/finders/image.rb

@@ -0,0 +1,51 @@
+module Image
+
+  def image
+    @image ||= find_image.to_s.gsub('http://', 'https://')
+  end
+
+  protected
+
+  def find_image
+    log "Curation::Page find_image #{url}"
+    if json_ld.any?
+      json_ld.each do |ld|
+        ld = ld.first if ld.is_a?(Array)
+        if ld.has_key? 'image'
+          image_data = ld['image']
+          if image_data.is_a? String
+            log "Curation::Page find_image json_ld string"
+            return image_data 
+          end
+          if image_data.is_a? Array
+            first = image_data.first
+            if first.is_a? String
+              log "Curation::Page find_image json_ld array"
+              return first 
+            end
+            if first.is_a? Hash
+              log "Curation::Page find_image json_ld array url"
+              return first['url'] 
+            end
+          end
+          if image_data.is_a? Hash
+            log "Curation::Page find_image json_ld url"
+            return image_data['url'] 
+          end
+        end
+      end
+    end
+    begin
+      [
+        metainspector.images.best,
+        nokogiri.css('[property="og:image"]').first&.attributes['content'].value
+      ].each do |possibility|
+        return possibility unless possibility.to_s.empty?
+      end
+    rescue
+      puts 'Curation::Page find_image error'
+    end
+    return ''
+  end
+
+end

data/lib/curation/finders/publication_date.rb

@@ -0,0 +1,43 @@
+module PublicationDate
+
+  def date
+    @date ||= find_date
+  end
+
+  protected
+
+  def find_date
+    if json_ld.any?
+      json_ld.each do |ld|
+        next unless ['NewsArticle', 'ReportageNewsArticle'].include? ld['@type']
+        return Date.parse ld['datePublished'] if ld.has_key? 'datePublished'
+      end
+    end
+    return Date.parse metatags['date'] rescue nil
+    return Date.parse metatags['pubdate'] rescue nil
+    return Date.parse nokogiri.css('meta[property="article:published"]').first['content'] rescue nil
+    return Date.parse nokogiri.css('meta[property="article:published_time"]').first['content'] rescue nil
+    return Date.parse nokogiri.css('meta[property="og:article:published_time"]').first['content'] rescue nil
+    chunks = html.split('DisplayDate')
+    if chunks.count > 1
+      value = chunks[1]
+      value = value.split(',').first
+      value = value.gsub('"', '')
+      value = value[1..-1] if value[0] == ':'
+      return Date.parse value rescue nil
+    end
+    begin
+      value = nokogiri.css('.postDate').first
+      value = value.inner_text
+      value = value.gsub(' — ', '')
+      return Date.parse value
+    rescue
+    end
+    begin
+      value = nokogiri.css('.gta_post_date').first
+      value = value.inner_text
+      return Date.parse value
+    rescue
+    end
+  end
+end

data/lib/curation/finders/text.rb

@@ -0,0 +1,78 @@
+module Text
+
+  BLACKLIST = [
+    'head', 'script', 'style', 'iframe', 'nav', 'noscript', 'header', 'footer', 'aside',
+    '.navigation', '.top-menu-container', '.navbar', '.navbar-header', '.breadcrumb',
+    '#breadcrumbs', '[typeof="v:Breadcrumb"]', '.skip-link', '.search', '.search-form',
+    '.categories', '.post-categories', '.datas', '.post-datas', '.twitter-media',
+    '.instagram-media', '.widget', '.related-post-tags', '.social-list', '.top-scroll',
+    '.comments', '.signature', '.publicite', '.footer', '.Footer', '.footer-copyright',
+    '[itemprop*="author"]', '[style*="display:none;"]', '[style*="display:none"]',
+    '[style*="display: none;"]', '[style*="display: none"]', '[aria-hidden="true"]'
+  ]
+
+  def text
+    # require 'byebug'; byebug
+    @text ||= find_text
+  end
+
+  protected
+
+  def find_text
+    text = find_text_with_json_ld || find_text_with_nokogiri
+    text.to_s.dup.gsub!('<br><br>', '<br>')
+    # require 'byebug'; byebug
+    text = clean_encoding text
+    text
+  end
+
+  def find_text_with_json_ld
+    if json_ld.any?
+      json_ld.each do |ld|
+        next unless ['NewsArticle', 'ReportageNewsArticle'].include? ld['@type']
+        return ld['text'] if ld.has_key? 'text'
+        return ld['articleBody'] if ld.has_key? 'articleBody'
+      end
+    end
+    nil
+  end
+
+  def find_text_with_nokogiri
+    h = nokogiri.dup
+    h.xpath('//style').remove
+    BLACKLIST.each do |tag|
+      h.css(tag).remove
+    end
+    nodes = h.css('p')
+    if nodes.any? 
+      text = nodes.to_html
+      text
+    else
+      # Cleanup was too hard, let's try softer
+      h = nokogiri.dup
+      h.text
+    end
+  end
+
+  # r&Atilde;&copy;forme -> réforme
+  def clean_encoding(text)
+    clean_text = HTMLEntities.new.decode text
+    double_encoding = false
+    [
+      'é', # é
+      'è', # è
+      'î', # î
+      'ê', # ê
+    ].each do |string|
+      # require 'byebug'; byebug
+      double_encoding = true if clean_text.include? string
+    end
+    if double_encoding
+      clean_text.encode('iso-8859-1', undef: :replace)
+                .force_encoding('utf-8')
+    else
+      text
+    end
+  end
+
+end

data/lib/curation/finders/title.rb

@@ -0,0 +1,57 @@
+module Title
+
+  def title
+    @title ||= find_title.strip.gsub(/\s+/, ' ')
+  end
+
+  protected
+
+  def find_title
+    find_title_with_json_ld ||
+    find_title_with_metainspector ||
+    find_title_with_nokogiri ||
+    ''
+  end
+
+  def find_title_with_json_ld
+    if json_ld.any?
+      json_ld.each do |ld|
+        # require 'byebug'; byebug
+        ld = ld.first if ld.is_a?(Array)
+        return ld['headline'] if ld.has_key? 'headline'
+      end
+    end
+    nil
+  end
+
+  def find_title_with_metainspector
+    metainspector_best_title = metainspector.best_title
+    metainspector_title = metainspector.title
+    # Problème avec une balise <meta property="title" content="Run 0"  />,
+    # metainspector croit que c'est le titre de la page.
+    # Comme le title contient le best title, avec souvent des infos en plus sur le site, 
+    # on vérifie si le best title est bien contenu dans le title
+    if  metainspector_title.present? && 
+        metainspector_title.present? &&
+        metainspector_best_title.present? && 
+        metainspector_title.include?(metainspector_best_title)
+      return metainspector_best_title
+    elsif metainspector_title.present?
+      return metainspector_title
+    end
+  end
+
+  def find_title_with_nokogiri
+    begin
+      [
+        nokogiri.css('[itemprop="headline"]')&.first&.inner_text,
+        nokogiri.css('title')&.first&.inner_text
+      ].each do |possibility|
+        return possibility unless possibility.to_s.empty?
+      end
+    rescue
+      log 'Curation::Page find_title_with_nokogiri error'
+    end
+  end
+
+end

data/lib/curation/tools/jsonld.rb

@@ -0,0 +1,26 @@
+module Jsonld
+
+  def json_ld
+    unless defined?(@json_ld)
+      @json_ld = []
+      begin
+        options = nokogiri.css('[type="application/ld+json"]')
+        options.each do |option|
+          @json_ld << json_ld_from_object(option)
+        end
+        # Some sites have tables in tables
+        @json_ld.flatten!
+        # require 'byebug'; byebug
+      rescue
+        log 'Curation::Page json_ld error'
+      end
+    end
+    @json_ld
+  end
+
+  def json_ld_from_object(object)
+    JSON.parse object.inner_text
+  rescue
+    {}
+  end
+end

data/lib/curation/tools/metainspector.rb

@@ -0,0 +1,18 @@
+module Metainspector
+
+  def metainspector
+    unless @metainspector
+      @metainspector = html.nil?  ? MetaInspector.new(url)
+                                  : MetaInspector.new(url, document: html)
+    end
+    @metainspector
+  rescue
+    log 'Curation::Page metainspector error'
+  end
+
+  def metatags
+    @metatags ||= metainspector.meta_tag['name']
+  rescue
+    log 'Curation::Page metatags error'
+  end
+end

data/lib/curation/tools/nokogiri.rb

@@ -0,0 +1,17 @@
+module Nokogiri
+
+  def nokogiri
+    unless @nokogiri
+      if file.nil?
+        @nokogiri = metainspector.parsed
+      else
+        file.rewind
+        @nokogiri = Nokogiri::HTML file
+        file.rewind
+      end
+    end
+    @nokogiri
+  rescue
+    log 'Curation::Page nokogiri error'
+  end
+end

data/lib/curation/tools/raw.rb

@@ -0,0 +1,19 @@
+module Raw
+
+  def file
+    @file ||= URI.open url, 'User-Agent' => "Mozilla/5.0"
+  rescue
+    log "Curation::Page file error with url #{url}"
+  end
+
+  def html
+    unless @html
+      file.rewind
+      @html = file.read
+      file.rewind
+    end
+    @html
+  rescue
+    log "Curation::Page html error"
+  end
+end

data/lib/curation/version.rb

@@ -1,3 +1,3 @@
 module Curation
-  VERSION = "1.10"
+  VERSION = "2.0"
 end

data/lib/curation.rb

@@ -1,4 +1,12 @@
 require "curation/version"
+require "curation/tools/raw"
+require "curation/tools/nokogiri"
+require "curation/tools/jsonld"
+require "curation/tools/metainspector"
+require "curation/finders/image"
+require "curation/finders/publication_date"
+require "curation/finders/text"
+require "curation/finders/title"
 require "metainspector"
 require "open-uri"
 require "htmlentities"
@@ -7,257 +15,31 @@ module Curation
   class Error < StandardError; end
 
   class Page
-    attr_reader :url
+    # Tools
+    include Raw
+    include Jsonld
+    include Metainspector
+    include Nokogiri
+
+    # Finders
+    include Title
+    include Image
+    include PublicationDate
+    include Text
 
-    BLACKLIST = [
-      'head', 'script', 'style', 'iframe', 'nav', 'noscript', 'header', 'footer', 'aside',
-      '.navigation', '.top-menu-container', '.navbar', '.navbar-header', '.breadcrumb',
-      '#breadcrumbs', '[typeof="v:Breadcrumb"]', '.skip-link', '.search', '.search-form',
-      '.categories', '.post-categories', '.datas', '.post-datas', '.twitter-media',
-      '.instagram-media', '.widget', '.related-post-tags', '.social-list', '.top-scroll',
-      '.comments', '.signature', '.publicite', '.footer', '.Footer', '.footer-copyright',
-      '[itemprop*="author"]', '[style*="display:none;"]', '[style*="display:none"]',
-      '[style*="display: none;"]', '[style*="display: none"]', '[aria-hidden="true"]'
-    ]
+    attr_reader :url
+    attr_accessor :verbose
 
     def initialize(url, html = nil)
       @url = url.to_s.gsub('http://', 'https://')
       @html = html
-    end
-
-    def title
-      @title ||= find_title
-    end
-
-    def image
-      unless @image
-        @image = find_image
-        @image = @image.to_s.gsub('http://', 'https://')
-      end
-      @image
-    end
-
-    def text
-      # require 'byebug'; byebug
-      @text ||= find_text
-    end
-
-    def date
-      @date ||= find_date
+      @verbose = false
     end
 
     protected
 
-    def find_title
-      if json_ld.any?
-        json_ld.each do |ld|
-          # require 'byebug'; byebug
-          ld = ld.first if ld.is_a?(Array)
-          return ld['headline'] if ld.has_key? 'headline'
-        end
-      end
-      begin
-        [
-          metainspector.best_title,
-          metainspector.title,
-          nokogiri.css('[itemprop="headline"]')&.first&.inner_text,
-          nokogiri.css('title')&.first&.inner_text
-        ].each do |possibility|
-          return possibility unless possibility.to_s.empty?
-        end
-      rescue
-        puts 'Curation::Page find_title error'
-      end
-      return ''
-    end
-
-    def find_image
-      if json_ld.any?
-        json_ld.each do |ld|
-          ld = ld.first if ld.is_a?(Array)
-          if ld.has_key? 'image'
-            image_data = ld['image']
-            return image_data if image_data.is_a? String
-            if image_data.is_a? Array
-              first = image_data.first
-              return first if first.is_a? String
-              return first['url'] if first.is_a? Hash
-            end
-            return image_data['url'] if image_data.is_a? Hash
-          end
-        end
-      end
-      begin
-        [
-          metainspector.images.best,
-          nokogiri.css('[property="og:image"]').first&.attributes['content'].value
-        ].each do |possibility|
-          return possibility unless possibility.to_s.empty?
-        end
-      rescue
-        puts 'Curation::Page find_image error'
-      end
-      return ''
-    end
-
-    def find_text
-      text = find_text_with_json_ld || find_text_with_nokogiri
-      text.to_s.gsub!('<br><br>', '<br>')
-      # require 'byebug'; byebug
-      text = clean_encoding text
-      text
-    end
-
-    def find_text_with_json_ld
-      if json_ld.any?
-        json_ld.each do |ld|
-          next unless ['NewsArticle', 'ReportageNewsArticle'].include? ld['@type']
-          return ld['text'] if ld.has_key? 'text'
-          return ld['articleBody'] if ld.has_key? 'articleBody'
-        end
-      end
-      nil
-    end
-
-    def find_text_with_nokogiri
-      h = nokogiri.dup
-      BLACKLIST.each do |tag|
-        h.css(tag).remove
-      end
-      nodes = h.css('p')
-      nodes.xpath('//style').remove
-      text = nodes.to_html
-      text
-    end
-
-    def find_date
-      if json_ld.any?
-        json_ld.each do |ld|
-          next unless ['NewsArticle', 'ReportageNewsArticle'].include? ld['@type']
-          return Date.parse ld['datePublished'] if ld.has_key? 'datePublished'
-        end
-      end
-      return Date.parse metatags['date'] rescue nil
-      return Date.parse metatags['pubdate'] rescue nil
-      return Date.parse nokogiri.css('meta[property="article:published"]').first['content'] rescue nil
-      return Date.parse nokogiri.css('meta[property="article:published_time"]').first['content'] rescue nil
-      return Date.parse nokogiri.css('meta[property="og:article:published_time"]').first['content'] rescue nil
-      chunks = html.split('DisplayDate')
-      if chunks.count > 1
-        value = chunks[1]
-        value = value.split(',').first
-        value = value.gsub('"', '')
-        value = value[1..-1] if value[0] == ':'
-        return Date.parse value rescue nil
-      end
-      begin
-        value = nokogiri.css('.postDate').first
-        value = value.inner_text
-        value = value.gsub(' — ', '')
-        return Date.parse value
-      rescue
-      end
-      begin
-        value = nokogiri.css('.gta_post_date').first
-        value = value.inner_text
-        return Date.parse value
-      rescue
-      end
-    end
-
-    private
-
-    def json_ld
-      unless defined?(@json_ld)
-        @json_ld = []
-        begin
-          options = nokogiri.css('[type="application/ld+json"]')
-          options.each do |option|
-            @json_ld << json_ld_from_object(option)
-          end
-          # Some sites have tables in tables
-          @json_ld.flatten!
-          # require 'byebug'; byebug
-        rescue
-          puts 'Curation::Page json_ld error'
-        end
-      end
-      @json_ld
-    end
-
-    def json_ld_from_object(object)
-      JSON.parse object.inner_text
-    rescue
-      {}
-    end
-
-    def file
-      @file ||= URI.open url, 'User-Agent' => "Mozilla/5.0"
-    rescue
-      puts "Curation::Page file error with url #{url}"
-    end
-
-    def html
-      unless @html
-        file.rewind
-        @html = file.read
-        file.rewind
-      end
-      @html
-    rescue
-      puts "Curation::Page html error"
-    end
-
-    def nokogiri
-      unless @nokogiri
-        if file.nil?
-          @nokogiri = metainspector.parsed
-        else
-          file.rewind
-          @nokogiri = Nokogiri::HTML file
-          file.rewind
-        end
-      end
-      @nokogiri
-    rescue
-      puts 'Curation::Page nokogiri error'
-    end
-
-    def metainspector
-      unless @metainspector
-        @metainspector = html.nil?  ? MetaInspector.new(url)
-                                    : MetaInspector.new(url, document: html)
-      end
-      @metainspector
-    rescue
-      puts 'Curation::Page metainspector error'
-    end
-
-    def metatags
-      @metatags ||= metainspector.meta_tag['name']
-    rescue
-      puts 'Curation::Page metatags error'
-    end
-
-    # r&Atilde;&copy;forme -> réforme
-    def clean_encoding(text)
-      clean_text = HTMLEntities.new.decode text
-      double_encoding = false
-      [
-        'é', # é
-        'è', # è
-        'î', # î
-        'ê', # ê
-      ].each do |string|
-        # require 'byebug'; byebug
-        double_encoding = true if clean_text.include? string
-      end
-      if double_encoding
-        clean_text.encode('iso-8859-1', undef: :replace)
-                  .force_encoding('utf-8')
-      else
-        text
-      end
+    def log(message)
+      puts message if verbose
     end
   end
 end

metadata

@@ -1,29 +1,29 @@
 --- !ruby/object:Gem::Specification
 name: curation
 version: !ruby/object:Gem::Version
-  version: '1.10'
+  version: '2.0'
 platform: ruby
 authors:
 - Arnaud Levy
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2022-06-08 00:00:00.000000000 Z
+date: 2023-11-11 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: metainspector
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '5.12'
+        version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '5.12'
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: nokogiri
   requirement: !ruby/object:Gem::Requirement
@@ -72,13 +72,21 @@ files:
 - bin/setup
 - curation.gemspec
 - lib/curation.rb
+- lib/curation/finders/image.rb
+- lib/curation/finders/publication_date.rb
+- lib/curation/finders/text.rb
+- lib/curation/finders/title.rb
+- lib/curation/tools/jsonld.rb
+- lib/curation/tools/metainspector.rb
+- lib/curation/tools/nokogiri.rb
+- lib/curation/tools/raw.rb
 - lib/curation/version.rb
-homepage: https://github.com/arnaudlevy/curation
+homepage: https://github.com/noesya/curation
 licenses:
 - MIT
 metadata:
-  homepage_uri: https://github.com/arnaudlevy/curation
-  source_code_uri: https://github.com/arnaudlevy/curation
+  homepage_uri: https://github.com/noesya/curation
+  source_code_uri: https://github.com/noesya/curation
 post_install_message:
 rdoc_options: []
 require_paths:
@@ -94,7 +102,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.6
+rubygems_version: 3.4.10
 signing_key:
 specification_version: 4
 summary: Curation of content