earl 0.3.0 → 2.0.0

data/spec/integration/feed_spec.rb

@@ -0,0 +1,78 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+
+describe Earl do
+  subject(:instance) { Earl[url] }
+
+  vcr_base = 'feed'
+
+  context 'when page has no feeds associated', vcr: { cassette_name: "#{vcr_base}/no_feed" } do
+    let(:url) { 'https://www.google.com/' }
+    it { expect(subject.feed?).to be false }
+    it { expect(subject.rss_feed).to be_nil }
+    it { expect(subject.feed).to be_nil }
+  end
+
+  context 'when page has rss feed associated', vcr: { cassette_name: "#{vcr_base}/with_rss_feed" } do
+    let(:url) { 'https://rubyweekly.com/' }
+    it { expect(subject.feed?).to be true }
+    it { expect(subject.rss_feed).to eql('/rss/') }
+    it { expect(subject.feed).to eql('/rss/') }
+  end
+
+  # # context "when page has atom feed associated" do
+  # #   let(:url) { 'http:// still looking for a page that just has atom' }
+  # #    { expect(subject.feed?).to be true }
+  # #    { expect(subject.atom_feed).to eql('http://www.readwriteweb.com/atom.xml') }
+  # #    { expect(subject.feed).to eql('http://www.readwriteweb.com/atom.xml') }
+  # # end
+
+  context 'when page has rss and atom feed associated', vcr: { cassette_name: "#{vcr_base}/with_atom_and_rss_feed" } do
+    let(:url) { 'https://0xfe.blogspot.com/' }
+    let(:expected_rss_feed) { 'https://0xfe.blogspot.com/feeds/posts/default?alt=rss' }
+    let(:expected_atom_feed) { 'https://0xfe.blogspot.com/feeds/posts/default' }
+
+    it { expect(subject.feed?).to be true }
+    it { expect(subject.rss_feed).to eql(expected_rss_feed) }
+    it { expect(subject.atom_feed).to eql(expected_atom_feed) }
+    describe '#feed' do
+      context 'default (rss)' do
+        subject { instance.feed }
+        it { expect(subject).to eql(expected_rss_feed) }
+      end
+      context 'rss prefered' do
+        subject { instance.feed(:rss) }
+        it { expect(subject).to eql(expected_rss_feed) }
+      end
+      context 'atom prefered' do
+        subject { instance.feed(:atom) }
+        it { expect(subject).to eql(expected_atom_feed) }
+      end
+    end
+  end
+
+  context 'when page IS an rss feed', vcr: { cassette_name: "#{vcr_base}/is_rss_feed" } do
+    let(:url) { 'https://cprss.s3.amazonaws.com/rubyweekly.com.xml' }
+    it { expect(subject.url).to eql(url) }
+    it { expect(subject.base_url).to eql(url) }
+    it { expect(subject.content_type).to eql('application/xml') }
+    # TODO: fix rss feed detection
+    # it { expect(subject.feed?).to be true }
+    # it { expect(subject.rss_feed).to eql(url) }
+    # it { expect(subject.feed).to eql(url) }
+    it { expect(subject.feed?).to be false }
+    it { expect(subject.rss_feed).to be_nil }
+    it { expect(subject.feed).to be_nil }
+  end
+
+  context 'when page IS an atom feed', vcr: { cassette_name: "#{vcr_base}/is_atom_feed" } do
+    let(:url) { 'https://0xfe.blogspot.com/feeds/posts/default' }
+    it { expect(subject.url).to eql(url) }
+    it { expect(subject.base_url).to eql(url) }
+    it { expect(subject.content_type).to eql('application/atom+xml') }
+    it { expect(subject.feed?).to be true }
+    it { expect(subject.atom_feed).to eql(url) }
+    it { expect(subject.feed).to eql(url) }
+  end
+end

data/spec/integration/oembed_spec.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+
+describe Earl do
+  subject(:instance) { Earl[url] }
+
+  vcr_base = 'oembed'
+
+  context 'when page does not support oembed', vcr: { cassette_name: "#{vcr_base}/no_oembed" } do
+    let(:url) { 'https://github.com/evendis/earl' }
+    it { expect(subject.oembed).to be_nil }
+    it { expect(subject.oembed_html).to be_nil }
+    describe '#metadata' do
+      subject { instance.metadata }
+      it { expect(subject[:base_url]).to match(%r{github\.com/evendis/earl}) }
+      it { expect(subject[:content_type]).to eql('text/html') }
+      it { expect(subject[:html]).to be_nil }
+    end
+  end
+
+  context 'when youtube oembed', vcr: { cassette_name: "#{vcr_base}/youtube_oembed" } do
+    let(:url) { 'https://www.youtube.com/watch?v=hNSkCqMUMQA' }
+    let(:expected_oembed_html) { 'https://www.youtube.com/embed/hNSkCqMUMQA?feature=oembed' }
+
+    it { expect(subject.oembed).to be_a(Hash) }
+    it { expect(subject.oembed_html).to include(expected_oembed_html) }
+    describe '#metadata' do
+      subject { instance.metadata }
+      it { expect(subject[:base_url]).to eql(url) }
+      it { expect(subject[:content_type]).to eql('text/html') }
+      it { expect(subject[:title]).to eql('[JA][Keynote] Ruby Taught Me About Encoding Under the Hood / Mari Imaizumi @ima1zumi') }
+      it { expect(subject[:html]).to include(expected_oembed_html) }
+    end
+  end
+end

data/spec/spec_helper.rb

@@ -1,34 +1,26 @@
-require 'earl'
-
-# Configure RSpec
-RSpec.configure do |config|
-  # Use color
-  config.color_enabled = true
-  # Change the formatter
-  config.formatter = :documentation
-end
+# frozen_string_literal: true
 
-# Custom entity manipulation matcher
-RSpec::Matchers.define :produce do |expected|
+require 'earl'
 
-  match do |entity_class|
-    entity = entity_class.new @start
-    @changes.each do |key, value|
-      entity.send :"#{key}=", value
-    end
-    @actual = entity.to_s
-    @actual == expected
-  end
+# Requires supporting files with custom matchers and macros, etc,
+# in ./support/ and its subdirectories.
+Dir["#{File.dirname(__FILE__)}/support/**/*.rb"].each { |f| require f }
 
-  failure_message_for_should do |entity|
-    %Q{expected that "#{@start}" would become "#{expected}" when given #{@changes}, but was "#{@actual}"}
-  end
+RSpec.configure do |config|
+  # == Mock Framework
+  #
+  # If you prefer to use mocha, flexmock or RR, uncomment the appropriate line:
+  #
+  # config.mock_with :mocha
+  # config.mock_with :flexmock
+  # config.mock_with :rr
+  config.mock_with :rspec
 
-  chain :from do |start|
-    @start = start
-  end
+  # Remove this line if you're not using ActiveRecord or ActiveRecord fixtures
+  # config.fixture_path = "#{::Rails.root}/spec/fixtures"
 
-  chain :when_given do |changes|
-    @changes = changes
-  end
-end
+  # If you're not using ActiveRecord, or you'd prefer not to run each of your
+  # examples within a transaction, remove the following line or assign false
+  # instead of true.
+  # config.use_transactional_fixtures = true
+end

data/spec/support/fixtures.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+module FixtureHelper
+  def fixture_path(name)
+    File.dirname(__FILE__) + "/../fixtures/#{name}.html"
+  end
+
+  def fixture(name)
+    File.new(fixture_path(name)).read
+  end
+end
+
+RSpec.configure do |conf|
+  conf.include FixtureHelper
+end

data/spec/support/vcr.rb

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+require 'vcr'
+
+VCR.configure do |config|
+  config.cassette_library_dir = 'spec/fixtures/cassettes'
+  config.hook_into :webmock
+  config.configure_rspec_metadata!
+end

data/spec/unit/earl/earl_spec.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+
+describe Earl do
+  describe '.new' do
+    subject { Earl.new('http://test.host/') }
+    it { expect(subject.to_s).to eql('http://test.host/') }
+  end
+
+  describe '[]=' do
+    subject { Earl['http://test.host/'] }
+    it { expect(subject.to_s).to eql('http://test.host/') }
+  end
+end

data/spec/unit/earl/feed_spec.rb

@@ -0,0 +1,62 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+
+describe Earl do
+  let(:instance) { Earl[url] }
+  let(:url) { 'http:://example.com' }
+
+  before do
+    allow_any_instance_of(Earl).to receive(:uri_response).and_return(source)
+  end
+  subject { instance }
+
+  context 'when page has no feeds associated' do
+    let(:source) { fixture(:page_without_feeds) }
+    it { expect(subject.feed?).to be false }
+    it { expect(subject.rss_feed).to be_nil }
+    it { expect(subject.feed).to be_nil }
+  end
+
+  context 'when page has rss feed associated' do
+    let(:source) { fixture(:page_with_rss_feed) }
+    it { expect(subject.feed?).to be true }
+    it { expect(subject.rss_feed).to eql('http://www.readwriteweb.com/rss.xml') }
+    it { expect(subject.feed).to eql('http://www.readwriteweb.com/rss.xml') }
+  end
+
+  context 'when page has atom feed associated' do
+    let(:source) { fixture(:page_with_atom_feed) }
+    it { expect(subject.feed?).to be true }
+    it { expect(subject.atom_feed).to eql('http://www.readwriteweb.com/atom.xml') }
+    it { expect(subject.feed).to eql('http://www.readwriteweb.com/atom.xml') }
+  end
+
+  context 'when page has rss and atom feed associated' do
+    let(:source) { fixture(:page_with_rss_and_atom_feeds) }
+    it { expect(subject.feed?).to be true }
+    it { expect(subject.rss_feed).to eql('http://www.readwriteweb.com/rss.xml') }
+    it { expect(subject.atom_feed).to eql('http://www.readwriteweb.com/atom.xml') }
+    describe '#feed' do
+      context 'default (rss)' do
+        subject { instance.feed }
+        it { expect(subject).to eql('http://www.readwriteweb.com/rss.xml') }
+      end
+      context 'rss prefered' do
+        subject { instance.feed(:rss) }
+        it { expect(subject).to eql('http://www.readwriteweb.com/rss.xml') }
+      end
+      context 'atom prefered' do
+        subject { instance.feed(:atom) }
+        it { expect(subject).to eql('http://www.readwriteweb.com/atom.xml') }
+      end
+    end
+  end
+
+  # context "when page IS an rss feed" do
+  #   let(:source) { fixture(:page_as_rss) }
+  #   its(:feed?) { should be_true }
+  #   its(:rss_feed) { should eql(url) }
+  #   its(:feed) { should eql(url) }
+  # end
+end

data/spec/unit/earl/oembed_spec.rb

@@ -0,0 +1,50 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+
+class MockOembedResponse
+  def fields
+    { 'html' => '<iframe/>' }
+  end
+end
+
+describe Earl do
+  let(:instance) { Earl.new(url, options) }
+  let(:url) { 'http:://example.com' }
+  let(:options) { {} }
+
+  describe '#oembed_options' do
+    subject { instance.oembed_options }
+    context 'with default options' do
+      let(:expected) { { maxwidth: '560', maxheight: '315' } }
+      it { expect(subject).to eql(expected) }
+    end
+    context 'with custom options' do
+      let(:options) { { oembed: { maxwidth: '260' } } }
+      let(:expected) { { maxwidth: '260', maxheight: '315' } }
+      it { expect(subject).to eql(expected) }
+    end
+    context 'with custom options passed to oembed' do
+      let(:expected) { { maxwidth: '360', maxheight: '315' } }
+      before do
+        allow(instance).to receive(:fetch_oembed).and_return(nil)
+        instance.oembed({ maxwidth: '360' })
+      end
+      it { expect(subject).to eql(expected) }
+    end
+  end
+
+  describe '#oembed' do
+    let(:dummy_response) { MockOembedResponse.new }
+    before do
+      expect(instance).to receive(:base_url).and_return('')
+      expect(instance).to receive(:fetch_oembed).and_return(dummy_response)
+    end
+    subject { instance.oembed }
+    it { expect(subject).to eql({ html: '<iframe/>' }) }
+    describe '#oembed_html' do
+      subject { instance.oembed_html }
+      it { expect(subject).to eql('<iframe/>') }
+    end
+  end
+end

data/spec/unit/earl/scraper_spec.rb

@@ -0,0 +1,49 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+
+describe Earl::Scraper do
+  before do
+    allow_any_instance_of(Earl).to receive(:uri_response).and_return(
+      <<-DOC
+      <!DOCTYPE html PUBLIC "-//W3C//DTD HTML 4.0 Transitional//EN" "http://www.w3.org/TR/REC-html40/loose.dtd">
+      <html></html>
+      DOC
+    )
+  end
+
+  describe 'When validating URLs' do
+    before do
+      class Earl::TestScraper < Earl::Scraper
+        match %r{^http://www\.test\.com/$}
+        define_attribute(:title) { |_response| :test_title }
+      end
+    end
+
+    it 'returns the result if the URL matches the scraper regexp' do
+      expect(Earl['http://www.test.com/'].title).to eql :test_title
+    end
+  end
+
+  describe 'When retrieving the response' do
+    it 'returns a Nokogiri document' do
+      expect(Earl['test'].response.css('html').size).to eql 1
+    end
+  end
+
+  describe 'Scraper inheritance' do
+    class SubScraper < Earl::Scraper
+      define_attribute :some_attribute do |doc|
+        doc
+      end
+    end
+
+    it 'inherits all attributes from its superclass' do
+      scraper = SubScraper.new('foo.bar')
+      expect(scraper.attributes).to include(:title)
+      expect(scraper.attributes).to include(:description)
+      expect(scraper.attributes).to include(:image)
+      expect(scraper.attributes).to include(:some_attribute)
+    end
+  end
+end

data/spec/unit/earl_spec.rb

@@ -0,0 +1,74 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+
+describe Earl do
+  before do
+    allow_any_instance_of(Earl).to receive(:uri_response).and_return(source)
+  end
+  let(:url) { 'http://mock.net' }
+  subject(:instance) { Earl[url] }
+
+  describe '#title' do
+    subject { instance.title }
+    context 'example with title tag' do
+      let(:source) { fixture(:bicycles) }
+      it { expect(subject).to eql('bicycles (bicycles) on Twitter') }
+    end
+    context 'youtube example with title tag' do
+      let(:source) { fixture(:youtube) }
+      it { expect(subject).to eql('YouTube - Symptoms of Swine Flu') }
+    end
+  end
+
+  describe '#description' do
+    subject { instance.description }
+    context 'example with description tag' do
+      let(:source) { fixture(:bicycles) }
+      it { expect(subject).to eql('I write business plans for a living and fiction to stay sane.  I also try my hand at all sorts of creative side projects: painting, game dev, whiskey drinking..') }
+    end
+    context 'example without description tag' do
+      let(:source) { fixture(:bicycles_without_description) }
+      it { expect(subject).to be_nil }
+    end
+  end
+
+  describe '#image' do
+    subject { instance.image }
+    context 'example with image tag' do
+      let(:source) { fixture(:bicycles) }
+      it { expect(subject).to eql('http://assets0.twitter.com/images/loader.gif') }
+    end
+    context 'example without image tag' do
+      let(:source) { fixture(:bicycles_without_images) }
+      it { expect(subject).to be_nil }
+    end
+  end
+
+  describe '#attributes' do
+    subject { instance.attributes }
+    let(:source) { fixture(:bicycles) }
+    it 'is an array of expected elements' do
+      expect(subject).to be_an(Array)
+      %i[description title image].each do |attribute|
+        expect(subject).to include(attribute)
+      end
+    end
+  end
+
+  describe '#metadata' do
+    subject { instance.metadata }
+    let(:source) { fixture(:bicycles) }
+    it { expect(subject).to be_a(Hash) }
+    it 'has the expected elements' do
+      expect(subject).to eql({
+        title: 'bicycles (bicycles) on Twitter',
+        description: 'I write business plans for a living and fiction to stay sane.  I also try my hand at all sorts of creative side projects: painting, game dev, whiskey drinking..',
+        image: 'http://assets0.twitter.com/images/loader.gif',
+        rss_feed: '/statuses/user_timeline/user_id.rss',
+        base_url: 'http://mock.net',
+        feed: '/statuses/user_timeline/user_id.rss'
+      })
+    end
+  end
+end

metadata

@@ -1,113 +1,141 @@
 --- !ruby/object:Gem::Specification
 name: earl
 version: !ruby/object:Gem::Version
-  version: 0.3.0
-  prerelease: 
+  version: 2.0.0
 platform: ruby
 authors:
-- Jeremy Ruppel
-autorequire: 
+- teejayvanslyke
+- Paul Gallagher
 bindir: bin
 cert_chain: []
-date: 2012-04-25 00:00:00.000000000 Z
+date: 2025-07-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  name: treetop
+  name: nokogiri
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.4.10
+        version: '1.18'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.4.10
+        version: '1.18'
 - !ruby/object:Gem::Dependency
-  name: rspec
+  name: ruby-oembed
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: 2.9.0
-  type: :development
+        version: 0.18.1
+  type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: 2.9.0
-description: What URI wishes it could look like
+        version: 0.18.1
+description: URL metadata API
 email:
-- jeremy.ruppel@gmail.com
+- tj@elctech.com
+- gallagher.paul@gmail.com
 executables: []
 extensions: []
 extra_rdoc_files: []
 files:
-- .gitignore
-- .rvmrc
+- ".github/workflows/ruby-tests.yml"
+- ".gitignore"
+- ".rubocop.yml"
+- ".rubocop_todo.yml"
+- ".ruby-gemset"
+- ".ruby-version"
 - Gemfile
+- Guardfile
 - LICENSE
 - README.md
 - Rakefile
 - earl.gemspec
 - lib/earl.rb
-- lib/earl/email_assembler.rb
-- lib/earl/email_entity.rb
-- lib/earl/email_parser.tt
-- lib/earl/entity_base.rb
-- lib/earl/hash_inquirer.rb
-- lib/earl/string_inquirer.rb
-- lib/earl/url_assembler.rb
-- lib/earl/url_entity.rb
-- lib/earl/url_parser.tt
+- lib/earl/earl.rb
+- lib/earl/scraper.rb
 - lib/earl/version.rb
-- spec/earl/earl_spec.rb
-- spec/earl/email_entity_spec.rb
-- spec/earl/email_parser_spec.rb
-- spec/earl/entity_base_spec.rb
-- spec/earl/hash_inquirer_spec.rb
-- spec/earl/string_inquirer_spec.rb
-- spec/earl/url_entity_spec.rb
-- spec/earl/url_parser_spec.rb
+- spec/fixtures/bicycles.html
+- spec/fixtures/bicycles_without_description.html
+- spec/fixtures/bicycles_without_images.html
+- spec/fixtures/cassettes/feed/is_atom_feed.yml
+- spec/fixtures/cassettes/feed/is_rss_feed.yml
+- spec/fixtures/cassettes/feed/no_feed.yml
+- spec/fixtures/cassettes/feed/with_atom_and_rss_feed.yml
+- spec/fixtures/cassettes/feed/with_rss_feed.yml
+- spec/fixtures/cassettes/oembed/no_oembed.yml
+- spec/fixtures/cassettes/oembed/youtube_oembed.yml
+- spec/fixtures/page_as_atom.html
+- spec/fixtures/page_as_rss.html
+- spec/fixtures/page_with_atom_feed.html
+- spec/fixtures/page_with_rss_and_atom_feeds.html
+- spec/fixtures/page_with_rss_feed.html
+- spec/fixtures/page_without_feeds.html
+- spec/fixtures/youtube.html
+- spec/integration/feed_spec.rb
+- spec/integration/oembed_spec.rb
 - spec/spec_helper.rb
-homepage: https://github.com/remind101/earl
-licenses: []
-post_install_message: 
+- spec/support/fixtures.rb
+- spec/support/vcr.rb
+- spec/unit/earl/earl_spec.rb
+- spec/unit/earl/feed_spec.rb
+- spec/unit/earl/oembed_spec.rb
+- spec/unit/earl/scraper_spec.rb
+- spec/unit/earl_spec.rb
+homepage: https://github.com/evendis/earl
+licenses:
+- MIT
+metadata: {}
 rdoc_options: []
 require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
-  none: false
   requirements:
-  - - ! '>='
+  - - ">="
     - !ruby/object:Gem::Version
-      version: '0'
+      version: '3.0'
 required_rubygems_version: !ruby/object:Gem::Requirement
-  none: false
   requirements:
-  - - ! '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 1.8.19
-signing_key: 
-specification_version: 3
-summary: What URI wishes it could look like
+rubygems_version: 3.6.2
+specification_version: 4
+summary: URL metadata API for scraping titles, descriptions, images, and videos from
+  URLs
 test_files:
-- spec/earl/earl_spec.rb
-- spec/earl/email_entity_spec.rb
-- spec/earl/email_parser_spec.rb
-- spec/earl/entity_base_spec.rb
-- spec/earl/hash_inquirer_spec.rb
-- spec/earl/string_inquirer_spec.rb
-- spec/earl/url_entity_spec.rb
-- spec/earl/url_parser_spec.rb
+- spec/fixtures/bicycles.html
+- spec/fixtures/bicycles_without_description.html
+- spec/fixtures/bicycles_without_images.html
+- spec/fixtures/cassettes/feed/is_atom_feed.yml
+- spec/fixtures/cassettes/feed/is_rss_feed.yml
+- spec/fixtures/cassettes/feed/no_feed.yml
+- spec/fixtures/cassettes/feed/with_atom_and_rss_feed.yml
+- spec/fixtures/cassettes/feed/with_rss_feed.yml
+- spec/fixtures/cassettes/oembed/no_oembed.yml
+- spec/fixtures/cassettes/oembed/youtube_oembed.yml
+- spec/fixtures/page_as_atom.html
+- spec/fixtures/page_as_rss.html
+- spec/fixtures/page_with_atom_feed.html
+- spec/fixtures/page_with_rss_and_atom_feeds.html
+- spec/fixtures/page_with_rss_feed.html
+- spec/fixtures/page_without_feeds.html
+- spec/fixtures/youtube.html
+- spec/integration/feed_spec.rb
+- spec/integration/oembed_spec.rb
 - spec/spec_helper.rb
+- spec/support/fixtures.rb
+- spec/support/vcr.rb
+- spec/unit/earl/earl_spec.rb
+- spec/unit/earl/feed_spec.rb
+- spec/unit/earl/oembed_spec.rb
+- spec/unit/earl/scraper_spec.rb
+- spec/unit/earl_spec.rb