metainspector 5.18.1 → 5.20.0

data/spec/meta_inspector/images_spec.rb

@@ -70,6 +70,31 @@ describe MetaInspector do
   end
 
   describe "images.best" do
+    it 'should prefer the Parsely image over the largest image' do
+      html = '<meta name="parsely-image-url" content="/parsely.jpg"><img src="/largest.jpg" width="100" height="100">'
+      page = MetaInspector.new('http://example.com/', document: html)
+
+      expect(page.images.best).to eq('http://example.com/parsely.jpg')
+    end
+
+    ['', '  &#10;  '].each do |value|
+      it "should fall back to the largest image when the Parsely image is empty (#{value.inspect})" do
+        html = %(<meta name="parsely-image-url" content="#{value}"><img src="/largest.jpg" width="100" height="100">)
+        page = MetaInspector.new('http://example.com/', document: html)
+
+        expect(page.images.best).to eq('http://example.com/largest.jpg')
+      end
+    end
+
+    it "should find a valid image when another image URL cannot be parsed" do
+      html = '<html><body><img src="http://exa mple.com/x.png"><img src="/100x100"></body></html>'
+      page = MetaInspector.new('http://example.com/', document: html)
+
+      allow(FastImage).to receive(:size).and_call_original
+      expect(FastImage).not_to receive(:size).with(nil)
+      expect(page.images.best).to eq("http://example.com/100x100")
+    end
+
     it "should find the og image" do
       page = MetaInspector.new('http://www.theonion.com/articles/apple-claims-new-iphone-only-visible-to-most-loyal,2772/')
 
@@ -103,6 +128,44 @@ describe MetaInspector do
   end
 
   describe "images.owner_suggested" do
+    it 'should find the Parsely image when no existing metadata is present' do
+      html = '<meta name="parsely-image-url" content="https://images.example.com/parsely.jpg?b=2&amp;a=1">'
+      page = MetaInspector.new('http://example.com/', document: html)
+
+      expect(page.images.owner_suggested).to eq('https://images.example.com/parsely.jpg?b=2&a=1')
+    end
+
+    it 'should resolve a relative Parsely image against the document base URL' do
+      html = '<base href="/assets/"><meta name="parsely-image-url" content="parsely.jpg">'
+      page = MetaInspector.new('http://example.com/articles/page', document: html)
+
+      expect(page.images.owner_suggested).to eq('http://example.com/assets/parsely.jpg')
+    end
+
+    it 'should use the Parsely image when existing metadata is empty' do
+      html = '<meta property="og:image" content=""><meta name="twitter:image" content=""><meta name="parsely-image-url" content="/parsely.jpg">'
+      page = MetaInspector.new('http://example.com/', document: html)
+
+      expect(page.images.owner_suggested).to eq('http://example.com/parsely.jpg')
+    end
+
+    ['og:image', 'twitter:image'].each do |candidate|
+      it "should prefer #{candidate} over the Parsely image" do
+        html = %(<meta name="parsely-image-url" content="/parsely.jpg"><meta property="#{candidate}" content="/existing.jpg">)
+        page = MetaInspector.new('http://example.com/', document: html)
+
+        expect(page.images.owner_suggested).to eq('http://example.com/existing.jpg')
+      end
+    end
+
+    ['', '  &#10;  '].each do |value|
+      it "should ignore an empty Parsely image (#{value.inspect})" do
+        page = MetaInspector.new('http://example.com/', document: %(<meta name="parsely-image-url" content="#{value}">))
+
+        expect(page.images.owner_suggested).to be_nil
+      end
+    end
+
     it "should find the og image" do
       page = MetaInspector.new('http://www.theonion.com/articles/apple-claims-new-iphone-only-visible-to-most-loyal,2772/')
 
@@ -135,6 +198,176 @@ describe MetaInspector do
   end
 
   describe "images.with_size" do
+    describe "max_image_downloads" do
+      let(:html) do
+        '<img src="/10x10"><img src="/100x100"><img src="/remaining.jpg">'
+      end
+
+      before do
+        stub_request(:get, 'http://example.com/remaining.jpg').to_return(status: 404)
+      end
+
+      it "should download only the first images needing dimensions in document order" do
+        page = MetaInspector.new('http://example.com/', document: html, max_image_downloads: 2)
+
+        expect(page.images.with_size).to eq([
+          ['http://example.com/100x100', 100, 100],
+          ['http://example.com/10x10', 10, 10],
+          ['http://example.com/remaining.jpg', 0, 0]
+        ])
+        expect(WebMock).to have_requested(:get, 'http://example.com/10x10').once
+        expect(WebMock).to have_requested(:get, 'http://example.com/100x100').once
+        expect(WebMock).not_to have_requested(:get, 'http://example.com/remaining.jpg')
+      end
+
+      it "should preserve HTML dimensions without consuming the download budget" do
+        html = '<img src="/known.jpg" width="200" height="200"><img src="/10x10"><img src="/100x100"><img src="/later.jpg" width="300" height="300">'
+        page = MetaInspector.new('http://example.com/', document: html, max_image_downloads: 1)
+
+        expect(page.images.with_size).to eq([
+          ['http://example.com/later.jpg', 300, 300],
+          ['http://example.com/known.jpg', 200, 200],
+          ['http://example.com/10x10', 10, 10],
+          ['http://example.com/100x100', 0, 0]
+        ])
+        expect(WebMock).to have_requested(:get, 'http://example.com/10x10').once
+        expect(WebMock).not_to have_requested(:get, 'http://example.com/100x100')
+      end
+
+      it "should exclude invalid and duplicate URLs before consuming the budget" do
+        html = '<img src="http://exa mple.com/broken.jpg"><img src="/10x10"><img src="http://example.com/10x10"><img src="/100x100"><img src="/remaining.jpg">'
+        page = MetaInspector.new('http://example.com/', document: html, max_image_downloads: 2)
+
+        expect(page.images.with_size).to eq([
+          ['http://example.com/100x100', 100, 100],
+          ['http://example.com/10x10', 10, 10],
+          ['http://example.com/remaining.jpg', 0, 0]
+        ])
+        expect(WebMock).to have_requested(:get, 'http://example.com/10x10').once
+        expect(WebMock).to have_requested(:get, 'http://example.com/100x100').once
+        expect(WebMock).not_to have_requested(:get, 'http://example.com/remaining.jpg')
+      end
+
+      it "should treat an image missing one HTML dimension as needing a download" do
+        html = '<img src="/10x10" width="500"><img src="/100x100" height="500">'
+        page = MetaInspector.new('http://example.com/', document: html, max_image_downloads: 1)
+
+        expect(page.images.with_size).to eq([
+          ['http://example.com/10x10', 10, 10],
+          ['http://example.com/100x100', 0, 0]
+        ])
+        expect(WebMock).to have_requested(:get, 'http://example.com/10x10').once
+        expect(WebMock).not_to have_requested(:get, 'http://example.com/100x100')
+      end
+
+      it "should disable downloads with a zero limit while preserving HTML dimensions" do
+        html = '<img src="/10x10"><img src="/known.jpg" width="200" height="100">'
+        page = MetaInspector.new('http://example.com/', document: html, max_image_downloads: 0)
+
+        expect(page.images.with_size).to eq([
+          ['http://example.com/known.jpg', 200, 100],
+          ['http://example.com/10x10', 0, 0]
+        ])
+        expect(WebMock).not_to have_requested(:get, 'http://example.com/10x10')
+      end
+
+      it "should respect download_images: false even with a positive limit" do
+        page = MetaInspector.new('http://example.com/', document: html, download_images: false, max_image_downloads: 2)
+
+        expect(page.images.with_size).to eq([
+          ['http://example.com/10x10', 0, 0],
+          ['http://example.com/100x100', 0, 0],
+          ['http://example.com/remaining.jpg', 0, 0]
+        ])
+        expect(WebMock).not_to have_requested(:get, %r{http://example.com/})
+      end
+
+      [nil, 10].each do |limit|
+        it "should download every candidate when the limit is #{limit.inspect}" do
+          page = MetaInspector.new('http://example.com/', document: html, max_image_downloads: limit)
+
+          expect(page.images.with_size).to eq([
+            ['http://example.com/100x100', 100, 100],
+            ['http://example.com/10x10', 10, 10],
+            ['http://example.com/remaining.jpg', 0, 0]
+          ])
+          expect(WebMock).to have_requested(:get, 'http://example.com/10x10').once
+          expect(WebMock).to have_requested(:get, 'http://example.com/100x100').once
+          expect(WebMock).to have_requested(:get, 'http://example.com/remaining.jpg').once
+        end
+      end
+
+      [:not_found, :invalid_image, :timeout].each do |failure|
+        it "should count a #{failure} image attempt toward the limit" do
+          request = stub_request(:get, 'http://example.com/broken.jpg')
+          case failure
+          when :not_found then request.to_return(status: 404)
+          when :invalid_image then request.to_return(body: 'not an image')
+          when :timeout then request.to_timeout
+          end
+          html = '<img src="/broken.jpg"><img src="/100x100">'
+          page = MetaInspector.new('http://example.com/', document: html, max_image_downloads: 1)
+
+          expect(page.images.with_size).to eq([
+            ['http://example.com/broken.jpg', 0, 0],
+            ['http://example.com/100x100', 0, 0]
+          ])
+          expect(request).to have_been_requested.once
+          expect(WebMock).not_to have_requested(:get, 'http://example.com/100x100')
+        end
+      end
+
+      it "should reuse the limited results for repeated size and best-image queries" do
+        page = MetaInspector.new('http://example.com/', document: html, max_image_downloads: 1)
+
+        2.times do
+          expect(page.images.with_size.first).to eq(['http://example.com/10x10', 10, 10])
+          expect(page.images.largest).to eq('http://example.com/10x10')
+          expect(page.images.best).to eq('http://example.com/10x10')
+        end
+        expect(WebMock).to have_requested(:get, 'http://example.com/10x10').once
+        expect(WebMock).not_to have_requested(:get, 'http://example.com/100x100')
+      end
+
+      it "should keep the budget local to each document" do
+        2.times do
+          page = MetaInspector.new('http://example.com/', document: html, max_image_downloads: 1)
+
+          expect(page.images.with_size.first).to eq(['http://example.com/10x10', 10, 10])
+        end
+        expect(WebMock).to have_requested(:get, 'http://example.com/10x10').twice
+        expect(WebMock).not_to have_requested(:get, 'http://example.com/100x100')
+      end
+
+      it "should keep all image URLs without downloading them when enumerating images" do
+        page = MetaInspector.new('http://example.com/', document: html, max_image_downloads: 1)
+
+        expect(page.images.to_a).to eq([
+          'http://example.com/10x10',
+          'http://example.com/100x100',
+          'http://example.com/remaining.jpg'
+        ])
+        expect(WebMock).not_to have_requested(:get, %r{http://example.com/})
+      end
+    end
+
+    [true, false].each do |download_images|
+      it "should ignore unparseable image URLs with download_images: #{download_images}" do
+        html = '<html><body><img src="http://exa mple.com/x.png"><img src="/valid.png" width="100" height="100"></body></html>'
+        page = MetaInspector.new('https://example.com/', document: html, download_images: download_images)
+
+        expect(FastImage).not_to receive(:size).with(nil)
+        expect(page.images.with_size).to eq([["https://example.com/valid.png", 100, 100]])
+      end
+    end
+
+    it "should ignore unparseable image URLs even when HTML sizes are present" do
+      html = '<html><body><img src="http://exa mple.com/x.png" width="100" height="100"></body></html>'
+      page = MetaInspector.new('https://example.com/', document: html)
+
+      expect(page.images.with_size).to eq([])
+    end
+
     it "should return sorted by area array of [img_url, width, height] using html sizes" do
       page = MetaInspector.new('http://example.com/largest_image_in_html')
 

data/spec/meta_inspector/links_spec.rb

@@ -174,6 +174,28 @@ describe MetaInspector do
   end
 
   describe 'Relative links with relative base' do
+    [
+      ['assets/', 'https://example.com/posts/article', 'https://example.com/posts/assets/next.html'],
+      ['../assets/', 'https://example.com/posts/2026/article', 'https://example.com/posts/assets/next.html'],
+      ['assets/base.html', 'https://example.com/posts/article', 'https://example.com/posts/assets/next.html'],
+      ['assets/', 'https://example.com/posts/article/', 'https://example.com/posts/article/assets/next.html'],
+      ['/assets/', 'https://example.com:8443/posts/article', 'https://example.com:8443/assets/next.html']
+    ].each do |base_href, url, expected_link|
+      it "resolves base #{base_href.inspect} against #{url}" do
+        html = %(<html><head><base href="#{base_href}"></head><body><a href="next.html">Next</a></body></html>)
+        m = MetaInspector.new(url, document: html)
+
+        expect(m.links.all).to eq([expected_link])
+      end
+    end
+
+    it 'resolves image URLs against a document-relative base in a nested directory' do
+      html = '<html><head><base href="assets/"></head><body><img src="image.jpg"></body></html>'
+      m = MetaInspector.new('https://example.com/posts/article', document: html)
+
+      expect(m.images.to_a).to eq(['https://example.com/posts/assets/image.jpg'])
+    end
+
     it 'should get the relative links with relative directory base' do
       m = MetaInspector.new('http://relativewithrelativebase.com/relativedir')
       expect(m.links.all).to eq(['http://relativewithrelativebase.com/other/about',

data/spec/meta_inspector/meta_inspector_spec.rb

@@ -5,14 +5,65 @@ describe MetaInspector do
     expect(MetaInspector.new('http://example.com').class).to eq(MetaInspector::Document)
   end
 
-  it "cache request" do
-    # Creates a memory cache (a Hash that responds to #read, #write and #delete)
-    cache = Hash.new
-    def cache.read(k) self[k]; end
-    def cache.write(k, v) self[k] = v; end
+  describe "response caching" do
+    # A memory cache (a Hash that responds to #read and #write)
+    let(:cache) do
+      store = Hash.new
+      def store.read(k) self[k]; end
+      def store.write(k, v) self[k] = v; end
+      store
+    end
 
-    MetaInspector.new('http://example.com', faraday_http_cache: { store: cache })
+    # The deprecation warning prints once per process, so each example starts as a fresh process would
+    before { MetaInspector::Request.instance_variable_set(:@faraday_http_cache_deprecation_warned, nil) }
 
-    expect(cache.keys).not_to be_empty
+    it "caches responses with faraday_options[:faraday_http_cache]" do
+      MetaInspector.new('http://example.com', faraday_options: { faraday_http_cache: { store: cache } })
+
+      expect(cache.keys).not_to be_empty
+    end
+
+    it "does not warn when the cache is configured inside faraday_options" do
+      expect do
+        MetaInspector.new('http://example.com', faraday_options: { faraday_http_cache: { store: cache } })
+      end.not_to output.to_stderr
+    end
+
+    it "leaves the faraday_options hash it was given unchanged" do
+      cache_options   = { store: cache }
+      faraday_options = { faraday_http_cache: cache_options }
+
+      MetaInspector.new('http://example.com', faraday_options: faraday_options)
+
+      expect(faraday_options).to eq({ faraday_http_cache: { store: cache } })
+      expect(cache_options).to eq({ store: cache })
+    end
+
+    it "still caches responses with the deprecated top-level faraday_http_cache option" do
+      expect do
+        MetaInspector.new('http://example.com', faraday_http_cache: { store: cache })
+      end.to output.to_stderr
+
+      expect(cache.keys).not_to be_empty
+    end
+
+    it "warns that the top-level faraday_http_cache option is deprecated" do
+      expect do
+        MetaInspector.new('http://example.com', faraday_http_cache: { store: cache })
+      end.to output(/DEPRECATION.*faraday_http_cache.*faraday_options/).to_stderr
+    end
+
+    it "prints the deprecation warning only once per process" do
+      expect do
+        3.times { MetaInspector.new('http://example.com', faraday_http_cache: { store: cache }) }
+      end.to output(/\ADEPRECATION WARNING: [^\n]*\n\z/).to_stderr
+    end
+
+    it "raises ArgumentError when the cache is configured in both places" do
+      expect do
+        MetaInspector.new('http://example.com', faraday_http_cache: { store: cache },
+                                                faraday_options:    { faraday_http_cache: { store: cache } })
+      end.to raise_error(ArgumentError, /faraday_http_cache/)
+    end
   end
 end

data/spec/meta_inspector/redirections_spec.rb

@@ -52,22 +52,36 @@ describe MetaInspector do
     end
 
     context "when there is a callback to be ran between redirects that blocks redirections to private IP addresses" do
-      it "raises an exception" do
-        stub_request(:get, "https://www.facebook.com/")
-          .to_return(:status => 302,
-                     :headers => { "Location" => "http://10.0.0.0/" })
-
-        redirect_options = {
+      let(:redirect_options) do
+        {
           callback: proc do |_previous_response, next_request|
             ip_address = Resolv.getaddress(next_request.url.host)
             raise PrivateIPAddressError if IPAddr.new(ip_address).private?
           end
         }
+      end
+
+      before do
+        stub_request(:get, "https://www.facebook.com/")
+          .to_return(:status => 302,
+                     :headers => { "Location" => "http://10.0.0.0/" })
+      end
 
+      it "raises an exception" do
         expect {
           MetaInspector.new("https://www.facebook.com/", faraday_options: { redirect: redirect_options })
         }.to raise_error PrivateIPAddressError
       end
+
+      it "keeps running the callback when the same faraday_options are reused" do
+        faraday_options = { redirect: redirect_options }
+
+        2.times do
+          expect {
+            MetaInspector.new("https://www.facebook.com/", faraday_options: faraday_options)
+          }.to raise_error PrivateIPAddressError
+        end
+      end
     end
   end
 

data/spec/meta_inspector/texts_spec.rb

@@ -1,6 +1,50 @@
 require 'spec_helper'
 
 describe MetaInspector do
+  describe '#language' do
+    ['en', 'en-US', 'zh-Hant-TW'].each do |language|
+      it "should return the declared language #{language.inspect}" do
+        page = MetaInspector.new('http://example.com/', document: %(<html lang="#{language}"><body></body></html>))
+
+        expect(page.language).to eq(language)
+      end
+    end
+
+    it 'should strip surrounding whitespace from the declared language' do
+      page = MetaInspector.new('http://example.com/', document: "<html lang=\" \t en-US \n \"><body></body></html>")
+
+      expect(page.language).to eq('en-US')
+    end
+
+    [nil, '', " \t\n "].each do |value|
+      it "should return nil for a missing or blank language (#{value.inspect})" do
+        attribute = value.nil? ? '' : %( lang="#{value}")
+        page = MetaInspector.new('http://example.com/', document: "<html#{attribute}><body></body></html>")
+
+        expect(page.language).to be_nil
+      end
+    end
+
+    it 'should read the root language rather than a descendant language' do
+      page = MetaInspector.new('http://example.com/', document: '<html lang="en"><body lang="es"></body></html>')
+
+      expect(page.language).to eq('en')
+    end
+
+    it 'should not infer language from xml:lang, metadata, or descendants' do
+      html = '<html xml:lang="fr"><head><meta http-equiv="content-language" content="de"></head><body lang="es">Hello</body></html>'
+      page = MetaInspector.new('http://example.com/', document: html)
+
+      expect(page.language).to be_nil
+    end
+
+    it 'should return the declared language from a fetched document' do
+      page = MetaInspector.new('http://pagerankalert.com/')
+
+      expect(page.language).to eq('en')
+    end
+  end
+
   it "should get the title from the head section" do
     page = MetaInspector.new('http://example.com')
     expect(page.title).to eq('An example page')
@@ -49,6 +93,42 @@ describe MetaInspector do
   end
 
   describe '#best_title' do
+    it 'should use the Parsely title when no existing candidate is present' do
+      page = MetaInspector.new('http://example.com/', document: '<meta name="parsely-title" content="  A Parsely&#10; title  ">')
+
+      expect(page.best_title).to eq('A Parsely title')
+    end
+
+    it 'should use the Parsely title when existing candidates are empty' do
+      html = '<html><head><meta name="title" content=" "><meta property="og:title" content=""><title> </title><meta name="parsely-title" content="Parsely title"></head><body><h1> </h1></body></html>'
+      page = MetaInspector.new('http://example.com/', document: html)
+
+      expect(page.best_title).to eq('Parsely title')
+    end
+
+    [
+      ['<meta name="title" content="Existing title">', ''],
+      ['<meta property="og:title" content="Existing title">', ''],
+      ['<title>Existing title</title>', ''],
+      ['', '<title>Existing title</title>'],
+      ['', '<h1>Existing title</h1>']
+    ].each do |head, body|
+      it "should prefer #{head + body} over the Parsely title" do
+        html = %(<html><head><meta name="parsely-title" content="Parsely title">#{head}</head><body>#{body}</body></html>)
+        page = MetaInspector.new('http://example.com/', document: html)
+
+        expect(page.best_title).to eq('Existing title')
+      end
+    end
+
+    ['', '  &#10;  '].each do |value|
+      it "should ignore an empty Parsely title (#{value.inspect})" do
+        page = MetaInspector.new('http://example.com/', document: %(<meta name="parsely-title" content="#{value}">))
+
+        expect(page.best_title).to be_nil
+      end
+    end
+
     it "should find 'head title' when that's the only thing" do
       page = MetaInspector.new('http://example.com/title_in_head')
       expect(page.best_title).to eq('This title came from the head')
@@ -105,6 +185,41 @@ describe MetaInspector do
   end
 
   describe "#best_author" do
+    it 'should use the Parsely author when no existing candidate is present' do
+      page = MetaInspector.new('http://example.com/', document: '<meta name="parsely-author" content="  A Parsely&#10; author  ">')
+
+      expect(page.best_author).to eq('A Parsely author')
+    end
+
+    it 'should use the Parsely author when existing candidates are empty' do
+      html = '<meta name="author" content=" "><meta name="twitter:creator" content=""><meta name="parsely-author" content="Parsely author"><a rel="author"> </a><address> </address>'
+      page = MetaInspector.new('http://example.com/', document: html)
+
+      expect(page.best_author).to eq('Parsely author')
+    end
+
+    [
+      '<meta name="author" content="Existing author">',
+      '<a rel="author">Existing author</a>',
+      '<address>Existing author</address>',
+      '<meta name="twitter:creator" content="Existing author">'
+    ].each do |candidate|
+      it "should prefer #{candidate} over the Parsely author" do
+        html = '<meta name="parsely-author" content="Parsely author">' + candidate
+        page = MetaInspector.new('http://example.com/', document: html)
+
+        expect(page.best_author).to eq('Existing author')
+      end
+    end
+
+    ['', '  &#10;  '].each do |value|
+      it "should ignore an empty Parsely author (#{value.inspect})" do
+        page = MetaInspector.new('http://example.com/', document: %(<meta name="parsely-author" content="#{value}">))
+
+        expect(page.best_author).to be_nil
+      end
+    end
+
     it "should return the author meta tag content if present" do
       page = MetaInspector.new('http://example.com/author_in_meta')
 

data/spec/request_spec.rb

@@ -68,6 +68,35 @@ describe MetaInspector::Request do
     end
   end
 
+  describe "options it is given" do
+    it "accepts a frozen faraday_options hash" do
+      expect do
+        MetaInspector::Request.new(url('http://example.com'), faraday_options: { ssl: { verify: true } }.freeze)
+      end.not_to raise_error
+    end
+
+    it "leaves the faraday_options hash unchanged" do
+      callback        = proc {}
+      faraday_options = { redirect: { callback: callback } }
+
+      MetaInspector::Request.new(url('http://example.com'), allow_redirections: true,
+                                                            faraday_options:    faraday_options)
+
+      expect(faraday_options).to eq({ redirect: { callback: callback } })
+    end
+
+    it "leaves the faraday_http_cache hash unchanged" do
+      cache = Hash.new
+      def cache.read(k) self[k]; end
+      def cache.write(k, v) self[k] = v; end
+      cache_options = { store: cache }
+
+      MetaInspector::Request.new(url('http://example.com'), faraday_options: { faraday_http_cache: cache_options })
+
+      expect(cache_options).to eq({ store: cache })
+    end
+  end
+
   describe 'exception handling' do
     before(:each) do
       WebMock.allow_net_connect!