earl 0.3.0 → 2.0.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: bfb59215c79763794ebf311cff3e939a0be46e4557abf8bbd8c358693b12e1ee
+  data.tar.gz: 550ea900108a04f4d12935bcd7fa398b95c886f59847c2492a7d90b96495f042
+SHA512:
+  metadata.gz: 16102c9eee3ff031411da1aacd80371a7474ea3ff5b70dad9a77ad83efe63a943db7b845e8b461a4c44de7685e8933c903a2c2511c41df00f69759cd06ece54c
+  data.tar.gz: 3b724772932a18d2de4f589959c28968fa9cc93cf158bca8ac4360166ae3b04877e8bb6dc09338329da94fbbccee4891819598f1e0a4a642d858d13362f9219f

data/.github/workflows/ruby-tests.yml

@@ -0,0 +1,32 @@
+name: Ruby Tests
+
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+    branches:
+      - main
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+
+    steps:
+    - name: Checkout code
+      uses: actions/checkout@v3
+
+    - name: Set up Ruby
+      uses: ruby/setup-ruby@v1
+      with:
+        ruby-version: 3.3
+        bundler-cache: true
+
+    - name: Install dependencies
+      run: bundle install
+
+    - name: Run tests
+      run: bundle exec rspec
+
+    - name: Run rubocop
+      run: bundle exec rubocop

data/.gitignore

@@ -15,3 +15,8 @@ spec/reports
 test/tmp
 test/version_tmp
 tmp
+*.bundle
+*.so
+*.o
+*.a
+mkmf.log

data/.rubocop.yml

@@ -0,0 +1,35 @@
+inherit_from: .rubocop_todo.yml
+
+AllCops:
+  NewCops: disable
+  SuggestExtensions: false
+  TargetRubyVersion: 3.0
+
+Layout/LineLength:
+  Max: 199
+
+Metrics/BlockLength:
+  Max: 63
+
+Metrics/ClassLength:
+  Max: 200
+
+Metrics/CyclomaticComplexity:
+  Max: 8
+
+Metrics/MethodLength:
+  Max: 20
+
+Style/ClassVars:
+  Exclude:
+    - 'lib/earl/scraper.rb'
+
+Style/Documentation:
+  Exclude:
+    - 'spec/**/*'
+
+Style/ClassAndModuleChildren:
+  EnforcedStyleForClasses: compact
+
+Layout/FirstHashElementIndentation:
+  EnforcedStyle: consistent

data/.rubocop_todo.yml

@@ -0,0 +1,22 @@
+# This configuration was generated by
+# `rubocop --auto-gen-config`
+# on 2025-07-29 10:15:32 UTC using RuboCop version 1.79.0.
+# The point is for the user to remove these configuration records
+# one by one as the offenses are removed from the code base.
+# Note that changes in the inspected code, or installation of new
+# versions of RuboCop, may require this file to be generated again.
+
+# Offense count: 7
+# This cop supports unsafe autocorrection (--autocorrect-all).
+# Configuration parameters: AllowSafeAssignment.
+Lint/AssignmentInCondition:
+  Exclude:
+    - 'lib/earl/earl.rb'
+    - 'lib/earl/scraper.rb'
+
+# Offense count: 2
+# Configuration parameters: AllowedMethods.
+# AllowedMethods: enums
+Lint/ConstantDefinitionInBlock:
+  Exclude:
+    - 'spec/unit/earl/scraper_spec.rb'

data/.ruby-gemset

@@ -0,0 +1 @@
+earl

data/.ruby-version

@@ -0,0 +1 @@
+3.3.3

data/Gemfile

@@ -1,4 +1,16 @@
+# frozen_string_literal: true
+
 source 'https://rubygems.org'
 
-# Specify your gem's dependencies in earl.gemspec
+# Specify gem dependencies in earl.gemspec
 gemspec
+
+# development dependencies
+gem 'bundler', '>= 2.2.33'
+gem 'guard-rspec'
+gem 'guard-rubocop'
+gem 'rake'
+gem 'rspec'
+gem 'rubocop', require: false
+gem 'vcr'
+gem 'webmock'

data/Guardfile

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+# A sample Guardfile
+# More info at https://github.com/guard/guard#readme
+
+guard :rspec, cmd: 'bundle exec rspec' do
+  watch(%r{^spec/.+_spec\.rb$})
+  watch('spec/spec_helper.rb') { 'spec' }
+  watch(%r{^lib/(.+)\.rb$}) { |m| "spec/unit/#{m[1]}_spec.rb" }
+end
+
+guard :rubocop, cli: ['--display-cop-names'] do
+  watch(/.+\.rb$/)
+  watch(%r{(?:.+/)?\.rubocop(?:_todo)?\.yml$}) { |m| File.dirname(m[0]) }
+end

data/LICENSE

@@ -1,4 +1,4 @@
-Copyright (c) 2012 Jeremy Ruppel
+Copyright (c) 2009 T.J. VanSlyke, 2012-2025 Paul Gallagher
 
 MIT License
 
@@ -19,4 +19,4 @@ MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
 NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
 LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
 OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
-WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -1,41 +1,143 @@
 # Earl
 
-What URI wishes it could look like.
+Earl wants to help you scrape all the relevant metadata for your favorite web pages so you can be as cool as
+Facebook when displaying user-submitted link content. Earl returns details like titles, descriptions, content type,
+associated feeds, and OEmbed definitions if available.
 
-## Installation
+Earl is based on an original source project called _earl_ by [teejayvanslyke](https://github.com/teejayvanslyke/earl) (but never released as a gem).
+The revamp was done by [Paul Gallagher](https://github.com/tardate), and master source is currently
+available at <https://github.com/evendis/earl>.
 
-Add this line to your application's Gemfile:
+The Earl gem is officially named _earl_. Big thanks go to [jeremyruppel](https://github.com/jeremyruppel) who
+contributed the ownership of the _earl_ gem name. The original _earl_ gem had a somewhat similar purpose - it is now defunct, but still available up to version 0.3.0 via rubgems. Any _earl_ gem with version 1.0.0 or higher is the new gem release (and is in no way backwardly compatible with
+earlier versions).
 
-    gem 'earl'
+## The Earl Cookbook
 
-And then execute:
+### How do instantiate Earl?
 
-    $ bundle
+Pass any url-like string to Earl:
 
-Or install it yourself as:
+    my_earl_instance = Earl.new('https://github.com/evendis/earl')
+    #
+    # or using the []= convenience method:
+    my_earl_instance = Earl['https://github.com/evendis/earl']
 
-    $ gem install earl
+### How do I inspect details of the page?
 
-## Usage
+    earl = Earl['https://github.com/evendis/earl']
+    earl.title
+    =>  "evendis/earl · GitHub"
+    earl.description
+    => "earl - URL metadata API for scraping titles, descriptions, images, and videos from URL's."
+    earl.image
+    => "https://a248.e.akamai.net/assets.github.com/images/modules/header/logov7@4x.png?1340935010"
 
-``` rb
-url = Earl::URL.new 'http://www.foo.com'
+### How do I get oembed details for a link?
 
-url.scheme         		# => 'http'
-url.scheme?        		# => true
+Earl will get oembed details for a link if they are available.
 
-url.subdomain      		# => 'www'
-url.subdomain.www? 		# => true
-url.subdomain.baz? 		# => false
+    earl = Earl['https://www.youtube.com/watch?v=hNSkCqMUMQA']
+    earl.oembed
+    => {:title=>"[JA][Keynote] Ruby Taught Me About Encoding Under the Hood / Mari Imaizumi @ima1zumi",
+        :author_name=>"RubyKaigi",
+        :author_url=>"https://www.youtube.com/@rubykaigi4884",
+        :type=>"video",
+        :height=>113,
+        :width=>200,
+        :version=>"1.0",
+        :provider_name=>"YouTube",
+        :provider_url=>"https://www.youtube.com/",
+        :thumbnail_height=>360,
+        :thumbnail_width=>480,
+        :thumbnail_url=>"https://i.ytimg.com/vi/hNSkCqMUMQA/hqdefault.jpg",
+        :html=>
+        "<iframe width=\"200\" height=\"113\" src=\"https://www.youtube.com/embed/hNSkCqMUMQA?feature=oembed\" frameborder=\"0\" allow=\"accelerometer; autoplay; clipboard-write; encrypted-media; gyroscope; picture-in-picture; web-share\" referrerpolicy=\"strict-origin-when-cross-origin\" allowfullscreen title=\"[JA][Keynote] Ruby Taught Me About Encoding Under the Hood / Mari Imaizumi @ima1zumi\"></iframe>"}
+    # to get the embed code:
+    earl.oembed_html
+    => "<iframe width=\"200\" height=\"113\" src=\"https://www.youtube.com/embed/hNSkCqMUMQA?feature=oembed\" frameborder=\"0\" allow=\"accelerometer; autoplay; clipboard-write; encrypted-media; gyroscope; picture-in-picture; web-share\" referrerpolicy=\"strict-origin-when-cross-origin\" allowfullscreen title=\"[JA][Keynote] Ruby Taught Me About Encoding Under the Hood / Mari Imaizumi @ima1zumi\"></iframe>"
 
-url.host = 'foo.edu'
-url.to_s           		# => 'http://www.foo.edu'
-```
+### How do I customise the oembed link?
 
-## Contributing
+Supported oembed parameters may be provided with to `Earl.new` or to the `oembed` call:
 
-1. Fork it
-2. Create your feature branch (`git checkout -b my-new-feature`)
-3. Commit your changes (`git commit -am 'Added some feature'`)
-4. Push to the branch (`git push origin my-new-feature`)
-5. Create new Pull Request
+    earl = Earl.new('https://www.youtube.com/watch?v=hNSkCqMUMQA', { oembed: { maxwidth: '200', maxheight: '320' }})
+    earl.oembed_html
+    => "<iframe width=\"200\" height=\"113\" src=\"https://www.youtube.com/embed/hNSkCqMUMQA?feature=oembed\" frameborder=\"0\" allow=\"accelerometer; autoplay; clipboard-write; encrypted-media; gyroscope; picture-in-picture; web-share\" referrerpolicy=\"strict-origin-when-cross-origin\" allowfullscreen title=\"[JA][Keynote] Ruby Taught Me About Encoding Under the Hood / Mari Imaizumi @ima1zumi\"></iframe>"
+
+### How do I inspect what attributes are available for a page?
+
+To see all of the attributes a URL provides, simply ask:
+
+    earl = Earl['https://github.com/evendis/earl']
+    earl.attributes
+    => [:title, :image, :description, :rss_feed, :atom_feed, :content_type, :base_url, :charset, :content_encoding, :headers, :feed]
+
+### How can I extend Earl to scrape additional page details?
+
+Need to scrape additional page details currently supported by Earl?  Implement your own scraper:
+
+    class QotdScraper < Earl::Scraper
+      match /^http\:\/\/www\.quotationspage\.com\/qotd\.html$/
+
+      define_attribute :qotd do |doc|
+        doc.at('dt.quote a').text
+      end
+    end
+
+The define_attribute method will supply you with a Nokogiri document which you can traverse to your heart's content.
+Use the match method to limit the scope of URLs that your scraper will apply to.
+
+Your new attribute is now available for use:
+
+    Earl['http://www.quotationspage.com/qotd.html'].qotd
+    => "Love is a snowmobile racing across the tundra and then suddenly it flips over, pinning you underneath. At night, the ice weasels come."
+
+### How do I install it for normal use?
+
+If using bundler, add gem 'earl' your application's Gemfile and run `bundle`.
+
+Or install it from the command-line:
+
+    gem install earl
+
+### How do I install it for gem development?
+
+To work on enhancements of fix bugs in Earl, fork and clone the github repository.
+If you are using bundler (recommended), run `bundle` to install development dependencies:
+
+    gem install bundler
+    bundle
+
+### How do I run the tests?
+
+Once development dependencies are installed, all unit tests are run with just:
+
+    $ rake
+    # or..
+    $ rake spec
+
+VCR is used to record integration tests. To re-record sessions, delete the corresponding cassette in
+[spec/fixtures/cassettes](./spec/fixtures/cassettes/).
+
+### How do I automatically run tests when I modify files?
+
+Guard is installed as part of the development dependencies. Start a guard process in a terminal window:
+
+    bundle exec guard
+
+It will run all the tests to start with by default. Then whenever you change a file, the associated tests will execute in this terminal window.
+
+## Contributing to Earl
+
+* Check out the latest master to make sure the feature hasn't been implemented or the bug hasn't been fixed yet
+* Check out the issue tracker to make sure someone already hasn't requested it and/or contributed it
+* Fork the project
+* Start a feature/bugfix branch
+* Commit and push until you are happy with your contribution
+* Make sure to add tests for it. This is important so I don't break it in a future version unintentionally.
+* Please try not to mess with the Rakefile, version, or history. If you want to have your own version, or is otherwise necessary, that is fine, but please isolate to its own commit so I can cherry-pick around it.
+
+## Copyright
+
+See [LICENSE](./LICENSE) for details.

data/Rakefile

@@ -1,5 +1,13 @@
-#!/usr/bin/env rake
+# frozen_string_literal: true
+
 require 'bundler/gem_tasks'
 require 'rspec/core/rake_task'
 
-RSpec::Core::RakeTask.new :spec
+RSpec::Core::RakeTask.new(:spec)
+
+task default: :spec
+
+desc 'Open an irb session preloaded with this library'
+task :console do
+  sh 'irb -I lib -r earl.rb'
+end

data/earl.gemspec

@@ -1,21 +1,26 @@
-# -*- encoding: utf-8 -*-
-require File.expand_path('../lib/earl/version', __FILE__)
+# frozen_string_literal: true
+
+require 'English'
+lib = File.expand_path('lib', __dir__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'earl/version'
 
 Gem::Specification.new do |gem|
-  gem.authors       = ["Jeremy Ruppel"]
-  gem.email         = ["jeremy.ruppel@gmail.com"]
-  gem.description   = %q{What URI wishes it could look like}
-  gem.summary       = %q{What URI wishes it could look like}
-  gem.homepage      = "https://github.com/remind101/earl"
+  gem.authors       = ['teejayvanslyke', 'Paul Gallagher']
+  gem.email         = ['tj@elctech.com', 'gallagher.paul@gmail.com']
+  gem.description   = 'URL metadata API'
+  gem.summary       = 'URL metadata API for scraping titles, descriptions, images, and videos from URLs'
+  gem.homepage      = 'https://github.com/evendis/earl'
 
-  gem.files         = `git ls-files`.split($\)
-  gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
+  gem.files         = `git ls-files`.split($OUTPUT_RECORD_SEPARATOR)
+  gem.executables   = gem.files.grep(%r{^bin/}).map { |f| File.basename(f) }
   gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
-  gem.name          = "earl"
-  gem.require_paths = ["lib"]
+  gem.name          = 'earl'
+  gem.require_paths = ['lib']
   gem.version       = Earl::VERSION
+  gem.license       = 'MIT'
 
-  gem.add_dependency 'treetop', '>= 1.4.10'
-
-  gem.add_development_dependency 'rspec', '>= 2.9.0'
+  gem.required_ruby_version = '>= 3.0'
+  gem.add_runtime_dependency 'nokogiri', '~> 1.18'
+  gem.add_runtime_dependency 'ruby-oembed', '~> 0.18.1'
 end

data/lib/earl/earl.rb

@@ -0,0 +1,172 @@
+# frozen_string_literal: true
+
+# Earl is a class that represents a URL and provides methods to fetch metadata about the page
+class Earl
+  attr_accessor :url, :options
+  attr_writer :oembed
+
+  def initialize(url, options = {})
+    @url = url
+    @options = options
+  end
+
+  def to_s
+    url
+  end
+
+  def uri
+    @uri ||= URI.parse(url)
+  end
+
+  def uri_response
+    @uri_response ||= uri.open
+  end
+
+  # Returns
+  # :base_url - the actual url (resolved after possible redirect) as a String
+  # :content_type - mime type
+  # :charset - returns a charset parameter in Content-Type field. It is downcased for canonicalization.
+  # :content_encoding - returns a list of encodings in Content-Encoding field as an Array of String. The encodings are downcased for canonicalization.
+  # :headers - raw response header metadata
+  # (excluded since this generally returns not RFC 2616 compliant date :last_modified  - returns a Time which represents Last-Modified field.
+  def uri_response_attribute(name)
+    case name
+    when :base_url
+      (uri_response_attribute(:base_uri) || url).to_s
+    when :headers
+      uri_response_attribute(:meta)
+    else
+      uri_response.respond_to?(name) && uri_response.send(name)
+    end
+  end
+  protected :uri_response_attribute
+
+  def uri_response_attributes
+    %i[content_type base_url charset content_encoding headers]
+  end
+  protected :uri_response_attributes
+
+  def scraper
+    @scraper ||= Scraper.for(url, self)
+  end
+
+  def response
+    scraper&.response
+  end
+
+  # Returns a hash of link meta data, including:
+  # :title, :description, :image (all attributes)
+  # :base_url
+  def metadata
+    data = oembed || {}
+    attributes.each do |attribute|
+      if attribute_value = send(attribute)
+        data[attribute] ||= attribute_value
+      end
+    end
+    data
+  end
+
+  def respond_to_missing?(name, include_private)
+    uri_response_attributes.include?(name) || scraper&.attribute?(name) || super
+  end
+
+  # Dispatch missing methods if a match for:
+  # - uri_response_attributes
+  # - scraper attributes
+  def method_missing(method, *args)
+    if uri_response_attributes.include?(method)
+      uri_response_attribute(method)
+    elsif scraper&.attribute?(method)
+      scraper.attribute(method)
+    else
+      super
+    end
+  end
+
+  # Returns a full array of attributes available for the link
+  def attributes
+    scraper.attributes.keys + uri_response_attributes + [:feed]
+  end
+
+  # Returns the options to be used for oembed
+  def oembed_options
+    { maxwidth: '560', maxheight: '315' }.merge(options[:oembed] || {})
+  end
+
+  # Returns the oembed meta data hash for the URL (or nil if not defined/available)
+  # e.g. for https://www.youtube.com/watch?v=hNSkCqMUMQA:
+  # {
+  #   :title=>"[JA][Keynote] Ruby Taught Me About Encoding Under the Hood / Mari Imaizumi @ima1zumi",
+  #   :author_name=>"RubyKaigi",
+  #   :author_url=>"https://www.youtube.com/@rubykaigi4884",
+  #   :type=>"video",
+  #   :height=>113,
+  #   :width=>200,
+  #   :version=>"1.0",
+  #   :provider_name=>"YouTube",
+  #   :provider_url=>"https://www.youtube.com/",
+  #   :thumbnail_height=>360,
+  #   :thumbnail_width=>480,
+  #   :thumbnail_url=>"https://i.ytimg.com/vi/hNSkCqMUMQA/hqdefault.jpg",
+  #   :html=> "<iframe width=\"200\" height=\"113\" src=\"https://www.youtube.com/embed/hNSkCqMUMQA?feature=oembed\" \
+  #     frameborder=\"0\" allow=\"accelerometer; autoplay; clipboard-write; encrypted-media; gyroscope; picture-in-picture; web-share\" \
+  #     referrerpolicy=\"strict-origin-when-cross-origin\" allowfullscreen \
+  #     title=\"[JA][Keynote] Ruby Taught Me About Encoding Under the Hood / Mari Imaizumi @ima1zumi\"></iframe>"
+  # }
+  #
+  # +options+ defines a custom oembed options hash and will cause a re-fetch of the oembed metadata
+  def oembed(options = nil)
+    if options # use custom options, refetch oembed metadata
+      @options[:oembed] = options
+      @oembed = nil
+    end
+    @oembed ||= begin
+      h = fetch_oembed(base_url).fields
+      if h
+        h.keys.each do |key| # symbolize_keys!
+          new_key = begin
+            key.to_sym
+          rescue StandardError
+            key
+          end
+          h[new_key] = h.delete(key)
+        end
+        h
+      end
+    rescue StandardError
+      nil
+    end
+  end
+
+  def fetch_oembed(base_url)
+    OEmbed::Providers.get(base_url)
+  end
+  protected :fetch_oembed
+
+  # Returns the oembed code for the url (or nil if not defined/available)
+  def oembed_html
+    oembed && oembed[:html]
+  end
+
+  # Returns true if there is an ATOM or RSS feed associated with this URL.
+  def feed?
+    !feed.nil?
+  end
+
+  # Returns the feed URL associated with this URL.
+  # Returns RSS by default, or ATOM if +prefer+ is not :rss.
+  def feed(prefer = :rss)
+    rss = rss_feed
+    atom = atom_feed
+    if rss && atom
+      prefer == :rss ? rss : atom
+    else
+      rss || atom
+    end
+  end
+
+  def self.[](url)
+    new(url)
+  end
+end

data/lib/earl/scraper.rb

@@ -0,0 +1,92 @@
+# frozen_string_literal: true
+
+# Base class for nokogiri page scraping
+class Earl::Scraper
+  class << self
+    @@registry = []
+    attr_reader :regexp, :attributes
+
+    def match(regexp)
+      @regexp = regexp
+      register self
+    end
+
+    def define_attribute(name, &block)
+      @attributes ||= {}
+      @attributes[name] = block
+    end
+
+    def for(url, earl_source)
+      @@registry.each do |klass|
+        return klass.new(url, earl_source) if klass.regexp.match(url)
+      end
+      Earl::Scraper.new(url, earl_source)
+    end
+
+    def register(scraper_klass)
+      @@registry << scraper_klass
+    end
+    private :register
+  end
+
+  attr_reader :earl_source
+
+  def initialize(url, earl_source = nil)
+    @url = url
+    @earl_source = earl_source
+  end
+
+  def response
+    @response ||= earl_source && Nokogiri::HTML(earl_source.uri_response)
+  end
+
+  def attribute(name)
+    return unless attribute?(name)
+
+    attributes[name].call(response)
+  end
+
+  def attributes
+    if self.class.superclass == Earl::Scraper
+      self.class.superclass.attributes.merge(self.class.attributes)
+    else
+      self.class.attributes
+    end
+  end
+
+  def attribute?(name)
+    return false unless self.class.attributes
+
+    attributes.key?(name)
+  end
+
+  define_attribute :title do |doc|
+    if title = doc.at('title')
+      title.content
+    end
+  end
+
+  define_attribute :image do |doc|
+    if first_image = doc.at('img')
+      first_image['src']
+    end
+  end
+
+  define_attribute :description do |doc|
+    if element = doc.at("meta[name='description']")
+      element['content']
+    end
+  end
+
+  define_attribute :rss_feed do |doc|
+    if element = doc.at("link[type='application/rss+xml']")
+      element['href']
+    end
+  end
+
+  define_attribute :atom_feed do |doc|
+    if element = doc.at("link[type='application/atom+xml']")
+      element['href']
+    end
+  end
+end

data/lib/earl/version.rb

@@ -1,3 +1,5 @@
-module Earl
-  VERSION = '0.3.0'
+# frozen_string_literal: true
+
+class Earl
+  VERSION = '2.0.0'
 end