curation 1.3

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 2127e0e8cd5eecb8611891be3ba3ddf434dab8e1c172eb2b4f2aaffa7ffa21e4
+  data.tar.gz: d0b84333c2e85798b4ac9d809d5e7a49b1fab0f9d2de2a88109ca9305ffbb2e6
+SHA512:
+  metadata.gz: b99857abd7cf08f321140261c00c613324209a2af09b668bc994a3a52704b2eef67d1986d6950a6347af5a093ef5e6174a5ddb486d7e6fe432de7402a33c01ab
+  data.tar.gz: 9dd921943b36746d21b55943ddb9105a6d20043a3de284d14da98bb0d5a97dce84f99649d2329f038393ae1b1c0c7d1e72f5a915f109955538c6d222e3388c9e

data/.gitignore

@@ -0,0 +1,8 @@
+/.bundle/
+/.yardoc
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/

data/CODE_OF_CONDUCT.md

@@ -0,0 +1,74 @@
+# Contributor Covenant Code of Conduct
+
+## Our Pledge
+
+In the interest of fostering an open and welcoming environment, we as
+contributors and maintainers pledge to making participation in our project and
+our community a harassment-free experience for everyone, regardless of age, body
+size, disability, ethnicity, gender identity and expression, level of experience,
+nationality, personal appearance, race, religion, or sexual identity and
+orientation.
+
+## Our Standards
+
+Examples of behavior that contributes to creating a positive environment
+include:
+
+* Using welcoming and inclusive language
+* Being respectful of differing viewpoints and experiences
+* Gracefully accepting constructive criticism
+* Focusing on what is best for the community
+* Showing empathy towards other community members
+
+Examples of unacceptable behavior by participants include:
+
+* The use of sexualized language or imagery and unwelcome sexual attention or
+advances
+* Trolling, insulting/derogatory comments, and personal or political attacks
+* Public or private harassment
+* Publishing others' private information, such as a physical or electronic
+  address, without explicit permission
+* Other conduct which could reasonably be considered inappropriate in a
+  professional setting
+
+## Our Responsibilities
+
+Project maintainers are responsible for clarifying the standards of acceptable
+behavior and are expected to take appropriate and fair corrective action in
+response to any instances of unacceptable behavior.
+
+Project maintainers have the right and responsibility to remove, edit, or
+reject comments, commits, code, wiki edits, issues, and other contributions
+that are not aligned to this Code of Conduct, or to ban temporarily or
+permanently any contributor for other behaviors that they deem inappropriate,
+threatening, offensive, or harmful.
+
+## Scope
+
+This Code of Conduct applies both within project spaces and in public spaces
+when an individual is representing the project or its community. Examples of
+representing a project or community include using an official project e-mail
+address, posting via an official social media account, or acting as an appointed
+representative at an online or offline event. Representation of a project may be
+further defined and clarified by project maintainers.
+
+## Enforcement
+
+Instances of abusive, harassing, or otherwise unacceptable behavior may be
+reported by contacting the project team at contact@arnaudlevy.com. All
+complaints will be reviewed and investigated and will result in a response that
+is deemed necessary and appropriate to the circumstances. The project team is
+obligated to maintain confidentiality with regard to the reporter of an incident.
+Further details of specific enforcement policies may be posted separately.
+
+Project maintainers who do not follow or enforce the Code of Conduct in good
+faith may face temporary or permanent repercussions as determined by other
+members of the project's leadership.
+
+## Attribution
+
+This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4,
+available at [https://contributor-covenant.org/version/1/4][version]
+
+[homepage]: https://contributor-covenant.org
+[version]: https://contributor-covenant.org/version/1/4/

data/Gemfile

@@ -0,0 +1,6 @@
+source "https://rubygems.org"
+
+# Specify your gem's dependencies in curation.gemspec
+gemspec
+
+gem "rake", "~> 12.0"

data/Gemfile.lock

@@ -0,0 +1,58 @@
+PATH
+  remote: .
+  specs:
+    curation (0.1.0)
+      metainspector
+      nokogiri
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    addressable (2.7.0)
+      public_suffix (>= 2.0.2, < 5.0)
+    domain_name (0.5.20190701)
+      unf (>= 0.0.5, < 1.0.0)
+    faraday (1.0.1)
+      multipart-post (>= 1.2, < 3)
+    faraday-cookie_jar (0.0.6)
+      faraday (>= 0.7.4)
+      http-cookie (~> 1.0.0)
+    faraday-encoding (0.0.5)
+      faraday
+    faraday-http-cache (2.2.0)
+      faraday (>= 0.8)
+    faraday_middleware (1.0.0)
+      faraday (~> 1.0)
+    fastimage (2.1.7)
+    http-cookie (1.0.3)
+      domain_name (~> 0.5)
+    metainspector (5.10.1)
+      addressable (~> 2.7.0)
+      faraday (~> 1.0.0)
+      faraday-cookie_jar (~> 0.0.6)
+      faraday-encoding (~> 0.0.5)
+      faraday-http-cache (~> 2.2.0)
+      faraday_middleware (~> 1.0.0)
+      fastimage (~> 2.1.7)
+      nesty (~> 1.0.2)
+      nokogiri (~> 1.10.9)
+    mini_portile2 (2.4.0)
+    multipart-post (2.1.1)
+    nesty (1.0.2)
+    nokogiri (1.10.10)
+      mini_portile2 (~> 2.4.0)
+    public_suffix (4.0.5)
+    rake (12.3.3)
+    unf (0.1.4)
+      unf_ext
+    unf_ext (0.0.7.7)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  curation!
+  rake (~> 12.0)
+
+BUNDLED WITH
+   2.1.4

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2020 Arnaud Levy
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,49 @@
+# Curation
+
+When you build content curation tools, you need to extract the content of pages (title, text, image...). This requires different strategies and some fine tuning to work efficiently.
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'curation'
+```
+
+And then execute:
+
+    $ bundle install
+
+Or install it yourself as:
+
+    $ gem install curation
+
+## Usage
+
+```
+page = Curation::Page.new url
+article = Article.new
+article.title = page.title
+article.text = page.text
+article.image = page.image
+article.save
+```
+
+## Development
+
+After checking out the repo, run `bin/setup` to install dependencies. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
+
+To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at https://github.com/[USERNAME]/curation. This project is intended to be a safe, welcoming space for collaboration, and contributors are expected to adhere to the [code of conduct](https://github.com/[USERNAME]/curation/blob/master/CODE_OF_CONDUCT.md).
+
+
+## License
+
+The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT).
+
+## Code of Conduct
+
+Everyone interacting in the Curation project's codebases, issue trackers, chat rooms and mailing lists is expected to follow the [code of conduct](https://github.com/[USERNAME]/curation/blob/master/CODE_OF_CONDUCT.md).

data/Rakefile

@@ -0,0 +1,2 @@
+require "bundler/gem_tasks"
+task :default => :spec

data/bin/console

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+
+require "bundler/setup"
+require "curation"
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+# require "pry"
+# Pry.start
+
+require "irb"
+IRB.start(__FILE__)

data/bin/setup

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+set -vx
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/curation.gemspec

@@ -0,0 +1,26 @@
+require_relative 'lib/curation/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "curation"
+  spec.version       = Curation::VERSION
+  spec.authors       = ["Arnaud Levy"]
+  spec.email         = ["contact@arnaudlevy.com"]
+
+  spec.summary       = 'Curation of content'
+  spec.description   = %q{When you build content curation tools, you need to extract the content of pages (title, text, image...). This requires different strategies and some fine tuning to work efficiently.}
+  spec.homepage      = "https://github.com/arnaudlevy/curation"
+  spec.license       = "MIT"
+  spec.required_ruby_version = Gem::Requirement.new(">= 2.3.0")
+
+  spec.metadata["homepage_uri"] = spec.homepage
+  spec.metadata["source_code_uri"] = "https://github.com/arnaudlevy/curation"
+
+  spec.files         = Dir.chdir(File.expand_path('..', __FILE__)) do
+    `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
+  end
+  spec.bindir        = "exe"
+  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.require_paths = ["lib"]
+  spec.add_dependency "metainspector"
+  spec.add_dependency "nokogiri"
+end

data/lib/curation.rb

@@ -0,0 +1,134 @@
+require "curation/version"
+require "metainspector"
+require "open-uri"
+
+module Curation
+  class Error < StandardError; end
+
+  class Page
+    attr_reader :url, :title, :text, :image
+
+    BLACKLIST = [
+      'head', 'script', 'style', 'iframe', 'nav', 'noscript', 'header', 'footer', 'aside',
+      '.navigation', '.top-menu-container', '.navbar', '.navbar-header', '.breadcrumb',
+      '#breadcrumbs', '[typeof="v:Breadcrumb"]', '.skip-link', '.search', '.search-form',
+      '.categories', '.post-categories', '.datas', '.post-datas', '.twitter-media',
+      '.instagram-media', '.widget', '.related-post-tags', '.social-list', '.top-scroll',
+      '.comments', '.signature', '.publicite', '.footer', '.Footer', '.footer-copyright',
+      '[itemprop*="author"]', '[style*="display:none;"]', '[style*="display:none"]',
+      '[style*="display: none;"]', '[style*="display: none"]', '[aria-hidden="true"]'
+    ]
+
+    def initialize(url, html = nil)
+      @url = url
+      @html = html
+    end
+
+    def title
+      @title = find_title
+    end
+
+    def image
+      @image = find_image
+      @image = @image.to_s.gsub('http://', 'https://')
+      @image
+    end
+
+    def text
+      if json_ld.any?
+        json_ld.each do |ld|
+          next unless ld['@type'] == 'NewsArticle'
+          return ld['text'] if ld.has_key? 'text'
+          return ld['articleBody'] if ld.has_key? 'articleBody'
+        end
+      end
+      h = nokogiri.dup
+      BLACKLIST.each do |tag|
+        h.css(tag).remove
+      end
+      nodes = h.css('p')
+      nodes.xpath('//style').remove
+      text = nodes.to_html
+      text.gsub!('<br><br>', '<br>')
+      text
+    end
+
+    protected
+
+    def find_title
+      if json_ld.any?
+        json_ld.each do |ld|
+          return ld['headline'] if ld.has_key? 'headline'
+        end
+      end
+      [
+        metainspector.best_title,
+        metainspector.title,
+        nokogiri.css('[itemprop="headline"]').first.inner_text,
+        nokogiri.css('title').first.inner_text
+      ].each do |possibility|
+        return possibility unless possibility.blank?
+      end
+      return ''
+    end
+
+    def find_image
+      if json_ld.any?
+        json_ld.each do |ld|
+          if ld.has_key? 'image'
+            image_data = ld['image']
+            return image_data if image_data.is_a? String
+            if image_data.is_a? Array
+              first = image_data.first
+              return first if first.is_a? String
+              return first['url'] if first.is_a? Hash
+            end
+            return image_data['url'] if image_data.is_a? Hash
+          end
+        end
+      end
+      [
+        metainspector.images.best,
+        nokogiri.css('[property="og:image"]').first&.attributes["content"].value
+      ].each do |possibility|
+        return possibility unless possibility.blank?
+      end
+      return ''
+    end
+
+    def json_ld
+      unless @json_ld
+        @json_ld = []
+        begin
+          options = nokogiri.css('[type="application/ld+json"]')
+          options.each do |option|
+            string = option.inner_text
+            hash = JSON.parse(string)
+            @json_ld << hash
+          end
+        rescue
+          puts "JSON LD error"
+        end
+      end
+      @json_ld
+    end
+
+    def html
+      @html ||= URI.open url
+    rescue
+      puts "Impossible to open #{url}"
+    end
+
+    def nokogiri
+      @nokogiri ||= Nokogiri::HTML html
+    rescue
+      puts "Nokogiri error"
+    end
+
+    def metainspector
+      @metainspector ||= MetaInspector.new url, document: html
+    rescue
+      puts "MetaInspector error"
+    end
+  end
+end

data/lib/curation/version.rb

@@ -0,0 +1,3 @@
+module Curation
+  VERSION = "1.3"
+end

metadata

@@ -0,0 +1,87 @@
+--- !ruby/object:Gem::Specification
+name: curation
+version: !ruby/object:Gem::Version
+  version: '1.3'
+platform: ruby
+authors:
+- Arnaud Levy
+autorequire: 
+bindir: exe
+cert_chain: []
+date: 2020-07-21 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: metainspector
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: nokogiri
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: When you build content curation tools, you need to extract the content
+  of pages (title, text, image...). This requires different strategies and some fine
+  tuning to work efficiently.
+email:
+- contact@arnaudlevy.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- CODE_OF_CONDUCT.md
+- Gemfile
+- Gemfile.lock
+- LICENSE.txt
+- README.md
+- Rakefile
+- bin/console
+- bin/setup
+- curation.gemspec
+- lib/curation.rb
+- lib/curation/version.rb
+homepage: https://github.com/arnaudlevy/curation
+licenses:
+- MIT
+metadata:
+  homepage_uri: https://github.com/arnaudlevy/curation
+  source_code_uri: https://github.com/arnaudlevy/curation
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 2.3.0
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.0.3
+signing_key: 
+specification_version: 4
+summary: Curation of content
+test_files: []