roadie 3.0.5 → 3.1.0.rc1

data/lib/roadie/markup_improver.rb

@@ -29,9 +29,10 @@ module Roadie
       ensure_declared_charset head
     end
 
-    private
+    protected
     attr_reader :dom
 
+    private
     def ensure_doctype_present
       return if uses_buggy_jruby?
       return if @html.include?('<!DOCTYPE ')

data/lib/roadie/net_http_provider.rb

@@ -0,0 +1,70 @@
+# encoding: UTF-8
+require 'set'
+require 'uri'
+require 'net/http'
+
+module Roadie
+  # @api public
+  # External asset provider that downloads stylesheets from some other server
+  # using Ruby's built-in {Net::HTTP} library.
+  #
+  # You can pass a whitelist of hosts that downloads are allowed on.
+  #
+  # @example Allowing all downloads
+  #   provider = Roadie::NetHttpProvider.new
+  #
+  # @example Only allowing your own app domains
+  #   provider = Roadie::NetHttpProvider.new(
+  #     whitelist: ["myapp.com", "assets.myapp.com", "www.myapp.com"]
+  #   )
+  class NetHttpProvider
+    attr_reader :whitelist
+
+    # @option options [Array<String>] :whitelist ([]) A list of host names that downloads are allowed from. Empty set means everything is allowed.
+    def initialize(options = {})
+      @whitelist = Array(options.fetch(:whitelist, [])).to_set
+    end
+
+    def find_stylesheet(url)
+      find_stylesheet!(url)
+    rescue CssNotFound
+      nil
+    end
+
+    def find_stylesheet!(url)
+      response = download(url)
+      if response.kind_of? Net::HTTPSuccess
+        Stylesheet.new url, response.body
+      else
+        raise CssNotFound.new(url, "Server returned #{response.code}: #{truncate response.body}", self)
+      end
+    rescue Timeout::Error
+      raise CssNotFound.new(url, "Timeout", self)
+    end
+
+    def to_s() inspect end
+    def inspect() "#<#{self.class} whitelist: #{whitelist.inspect}>" end
+
+    private
+    def download(url)
+      uri = URI.parse(url)
+      if access_granted_to?(uri.host)
+        Net::HTTP.get_response(uri)
+      else
+        raise CssNotFound.new(url, "#{uri.host} is not part of whitelist!", self)
+      end
+    end
+
+    def access_granted_to?(host)
+      whitelist.empty? || whitelist.include?(host)
+    end
+
+    def truncate(string)
+      if string.length > 50
+        string[0, 49] + "…"
+      else
+        string
+      end
+    end
+  end
+end

data/lib/roadie/null_provider.rb

@@ -7,6 +7,9 @@ module Roadie
     def find_stylesheet(name) empty_stylesheet end
     def find_stylesheet!(name) empty_stylesheet end
 
+    def to_s() inspect end
+    def inspect() "#<#{self.class}>" end
+
     private
     def empty_stylesheet() Stylesheet.new "(null)", "" end
   end

data/lib/roadie/path_rewriter_provider.rb

@@ -0,0 +1,64 @@
+module Roadie
+  # @api public
+  # This provider acts a bit like a pipeline in normal UNIX parlour by enabling
+  # you to make changes to the requested path. Some uses of this include:
+  #
+  #   * Convert absolute URLs into local filesystem paths.
+  #   * Convert between external DNS name into internal naming.
+  #   * Changing path structure of filenames.
+  #   * Removing digests from filenames.
+  #   * Handle query string logic.
+  #   * Skipping known-bad paths.
+  #
+  # There might be other useful things you could use it for. The basic premise
+  # is that a path is sent in to this provider, maybe modified and then passed
+  # on to the "upstream" provider (or {ProviderList}).
+  #
+  # If the block returns {nil} or {false}, the upstream provider will not be
+  # invoked and it will be treated as "not found". This makes it possible to
+  # use this provider as a filter only.
+  #
+  # @example Simple regex
+  #   provider = Roadie::PathRewriterProvider.new(other_provider) { |path|
+  #     path.gsub(/-[a-f0-9]+\.css$/, '.css')
+  #   }
+  #
+  # @example Filtering assets
+  #   # Only assets containing "email" in the path will be considered by other_provider
+  #   only_email_provider = Roadie::PathRewriterProvider.new(other_provider) { |path|
+  #     path =~ /email/ ? path : nil
+  #   }
+  #
+  # @example Handling "external" app assets as local assets
+  #   document.external_asset_providers = [
+  #     # Look for assets from "myapp.com" just like if we just specified a local path
+  #     Roadie::PathRewriterProvider.new(document.asset_providers) { |url|
+  #       uri = URI.parse(url)
+  #       uri.path if uri.host == "myapp.com"
+  #     },
+  #     # Any other asset should be downloaded like normal
+  #     Roadie::NetHttpProvider.new
+  #   ]
+  class PathRewriterProvider
+    attr_reader :provider, :filter
+
+    def initialize(provider, &filter)
+      @provider = provider
+      @filter = filter
+    end
+
+    def find_stylesheet(path)
+      new_path = filter.call(path)
+      provider.find_stylesheet(new_path) if new_path
+    end
+
+    def find_stylesheet!(path)
+      new_path = filter.call(path)
+      if new_path
+        provider.find_stylesheet!(new_path)
+      else
+        raise CssNotFound, "Filter returned #{new_path.inspect}"
+      end
+    end
+  end
+end

data/lib/roadie/provider_list.rb

@@ -9,7 +9,6 @@ module Roadie
   class ProviderList
     extend Forwardable
     include Enumerable
-    include AssetProvider
 
     # Wrap a single provider, or a list of providers into a {ProviderList}.
     #
@@ -31,6 +30,9 @@ module Roadie
       end
     end
 
+    # Returns a new empty list.
+    def self.empty() new([]) end
+
     def initialize(providers)
       @providers = providers
     end
@@ -44,6 +46,22 @@ module Roadie
       nil
     end
 
+    # Tries to find the given stylesheet and raises an {ProvidersFailed} error
+    # if no provider could find the asset.
+    #
+    # @return [Stylesheet]
+    def find_stylesheet!(name)
+      errors = []
+      @providers.each do |provider|
+        begin
+          return provider.find_stylesheet!(name)
+        rescue CssNotFound => error
+          errors << error
+        end
+      end
+      raise ProvidersFailed.new(name, self, errors)
+    end
+
     def to_s
       list = @providers.map { |provider|
         # Indent every line one level

data/lib/roadie/rspec.rb

@@ -1 +1,2 @@
 require 'roadie/rspec/asset_provider'
+require 'roadie/rspec/cache_store'

data/lib/roadie/rspec/cache_store.rb

@@ -0,0 +1,25 @@
+shared_examples_for "roadie cache store" do
+  it "allows storing Stylesheets" do
+    stylesheet = Roadie::Stylesheet.new("foo.css", "body { color: green; }")
+    expect(subject["foo"] = stylesheet).to eql stylesheet
+  end
+
+  it "allows retreiving stored stylesheets" do
+    stylesheet = Roadie::Stylesheet.new("foo.css", "body { color: green; }")
+    subject["foo"] = stylesheet
+    stored_stylesheet = subject["foo"]
+    expect(stored_stylesheet.to_s).to eq stylesheet.to_s
+  end
+
+  it "defaults to nil when cache does not contain path" do
+    expect(subject["bar"]).to be_nil
+  end
+
+  it "accepts nil assignments to clear cache" do
+    subject["foo"] = Roadie::Stylesheet.new("", "")
+    expect {
+      subject["foo"] = nil
+    }.to_not raise_error
+    expect(subject["foo"]).to be_nil
+  end
+end

data/lib/roadie/stylesheet.rb

@@ -20,6 +20,7 @@ module Roadie
     # @yield [selector, properties]
     # @yieldparam [Selector] selector
     # @yieldparam [Array<StyleProperty>] properties
+    # @deprecated Iterate over the #{blocks} instead. Will be removed on version 4.0.
     def each_inlinable_block(&block)
       # #map and then #each in order to support chained enumerations, etc. if
       # no block is provided

data/lib/roadie/url_generator.rb

@@ -64,9 +64,10 @@ module Roadie
       combine_segments(root_uri, base, path).to_s
     end
 
-    private
+    protected
     attr_reader :root_uri, :scheme
 
+    private
     def build_root_uri
       path = make_absolute url_options[:path]
       port = parse_port url_options[:port]

data/lib/roadie/utils.rb

@@ -1,5 +1,6 @@
 module Roadie
   module Utils
+    # @api private
     def path_is_absolute?(path)
       # Ruby's URI is pretty unforgiving, but roadie aims to be. Don't involve
       # URI for URLs that's easy to determine to be absolute.
@@ -15,6 +16,14 @@ module Roadie
         raise InvalidUrlPath.new(path, error)
       end
     end
+    # @api private
     module_function :path_is_absolute?
+
+    # @api private
+    def warn(message)
+      Kernel.warn("Roadie: #{message}")
+    end
+    # @api private
+    module_function :warn
   end
 end

data/lib/roadie/version.rb

@@ -1,3 +1,3 @@
 module Roadie
-  VERSION = '3.0.5'
+  VERSION = '3.1.0.rc1'
 end

data/roadie.gemspec

@@ -17,11 +17,12 @@ Gem::Specification.new do |s|
 
   s.required_ruby_version = ">= 1.9"
 
-  s.add_dependency 'nokogiri', '~> 1.6.0'
+  s.add_dependency 'nokogiri', '>= 1.5.0', '< 1.7.0'
   s.add_dependency 'css_parser', '~> 1.3.4'
 
   s.add_development_dependency 'rspec', '~> 3.0'
   s.add_development_dependency 'rspec-collection_matchers', '~> 1.0'
+  s.add_development_dependency 'webmock', '~> 1.21.0'
 
   s.extra_rdoc_files = %w[README.md Changelog.md]
   s.require_paths = %w[lib]

data/spec/hash_as_cache_store_spec.rb

@@ -0,0 +1,7 @@
+require "spec_helper"
+require "roadie/rspec"
+
+describe "Using Hash as a cache store" do
+  subject(:hash) { Hash.new }
+  it_behaves_like "roadie cache store"
+end

data/spec/integration_spec.rb

@@ -48,6 +48,51 @@ describe "Roadie functionality" do
     expect(result).to have_styling('color' => 'red').at_selector('p > em')
   end
 
+  it "stores styles that cannot be inlined in the <head>" do
+    document = Roadie::Document.new <<-HTML
+      <html>
+        <body>
+          <h1>Hello world!</h1>
+          <p>Check out these <em>awesome</em> prices!</p>
+        </body>
+      </html>
+    HTML
+    css = <<-CSS
+      em:hover { color: red; }
+      p:fung-shuei { color: spirit; }
+    CSS
+    document.add_css css
+    expect(Roadie::Utils).to receive(:warn).with(/fung-shuei/)
+
+    result = parse_html document.transform
+    expect(result).to have_selector("html > head > style")
+
+    styles = result.at_css("html > head > style").text
+    expect(styles).to include Roadie::Stylesheet.new("", css).to_s
+  end
+
+  it "can be configured to skip styles that cannot be inlined" do
+    document = Roadie::Document.new <<-HTML
+      <html>
+        <body>
+          <h1>Hello world!</h1>
+          <p>Check out these <em>awesome</em> prices!</p>
+        </body>
+      </html>
+    HTML
+    css = <<-CSS
+      em:hover { color: red; }
+      p:fung-shuei { color: spirit; }
+    CSS
+    document.add_css css
+    document.keep_uninlinable_css = false
+
+    expect(Roadie::Utils).to receive(:warn).with(/fung-shuei/)
+
+    result = parse_html document.transform
+    expect(result).to_not have_selector("html > head > style")
+  end
+
   it "inlines css from disk" do
     document = Roadie::Document.new <<-HTML
       <!DOCTYPE html>
@@ -85,6 +130,52 @@ describe "Roadie functionality" do
     expect { document.transform }.to_not raise_error
   end
 
+  it "ignores external css if no external providers are added" do
+    document = Roadie::Document.new <<-HTML
+      <!DOCTYPE html>
+      <html>
+        <head>
+          <title>Hello world!</title>
+          <link rel="stylesheet" href="http://example.com/big_em.css">
+        </head>
+        <body>
+          <h1>Hello world!</h1>
+          <p>Check out these <em>awesome</em> prices!</p>
+        </body>
+      </html>
+    HTML
+
+    document.external_asset_providers = []
+
+    result = parse_html document.transform
+    expect(result).to have_selector('head > link')
+    expect(result).to have_styling([]).at_selector('p > em')
+  end
+
+  it "inlines external css if configured" do
+    document = Roadie::Document.new <<-HTML
+      <!DOCTYPE html>
+      <html>
+        <head>
+          <title>Hello world!</title>
+          <link rel="stylesheet" href="http://example.com/big_em.css">
+        </head>
+        <body>
+          <h1>Hello world!</h1>
+          <p>Check out these <em>awesome</em> prices!</p>
+        </body>
+      </html>
+    HTML
+
+    document.external_asset_providers = TestProvider.new(
+      "http://example.com/big_em.css" => "em { font-size: 200%; }"
+    )
+
+    result = parse_html document.transform
+    expect(result).to have_styling('font-size' => '200%').at_selector('p > em')
+    expect(result).to_not have_selector('head > link')
+  end
+
   it "makes URLs absolute" do
     document = Roadie::Document.new <<-HTML
       <!DOCTYPE html>

data/spec/lib/roadie/asset_scanner_spec.rb

@@ -3,21 +3,23 @@ require 'spec_helper'
 
 module Roadie
   describe AssetScanner do
-    let(:provider) { TestProvider.new }
+    let(:normal_provider) { TestProvider.new }
+    let(:external_provider) { ProviderList.empty }
     let(:dom) { dom_document "<html></html>" }
 
     def dom_fragment(html); Nokogiri::HTML.fragment html; end
     def dom_document(html); Nokogiri::HTML.parse html; end
 
-    it "is initialized with a DOM tree and a asset provider set" do
-      scanner = AssetScanner.new dom, provider
+    it "is initialized with a DOM tree, a normal asset provider set, and an external asset provider set" do
+      scanner = AssetScanner.new dom, normal_provider, external_provider
       expect(scanner.dom).to eq(dom)
-      expect(scanner.asset_provider).to eq(provider)
+      expect(scanner.normal_asset_provider).to eq(normal_provider)
+      expect(scanner.external_asset_provider).to eq(external_provider)
     end
 
     describe "finding" do
       it "returns nothing when no stylesheets are referenced" do
-        scanner = AssetScanner.new dom, provider
+        scanner = AssetScanner.new dom, normal_provider, external_provider
         expect(scanner.find_css).to eq([])
       end
 
@@ -34,7 +36,7 @@ module Roadie
             </body>
           </html>
         HTML
-        scanner = AssetScanner.new dom, provider
+        scanner = AssetScanner.new dom, normal_provider, external_provider
 
         stylesheets = scanner.find_css
 
@@ -54,43 +56,56 @@ module Roadie
             </head>
           </html>
         HTML
-        scanner = AssetScanner.new dom, provider
+        scanner = AssetScanner.new dom, normal_provider, external_provider
         expect(scanner.find_css).to have(1).stylesheet
       end
 
-      it "finds referenced stylesheets through the provider" do
+      it "finds normal referenced stylesheets through the normal provider" do
         stylesheet = double "A stylesheet"
-        expect(provider).to receive(:find_stylesheet!).with("/some/url.css").and_return stylesheet
+        expect(normal_provider).to receive(:find_stylesheet!).with("/some/url.css").and_return stylesheet
 
         dom = dom_fragment %(<link rel="stylesheet" href="/some/url.css">)
-        scanner = AssetScanner.new dom, provider
+        scanner = AssetScanner.new dom, normal_provider, external_provider
+
+        expect(scanner.find_css).to eq([stylesheet])
+      end
+
+      it "finds external referenced stylesheets through the external provider" do
+        stylesheet = double "A stylesheet"
+        external_provider = TestProvider.new
+        expect(external_provider).to receive(:find_stylesheet!).with("//example.com/style.css").and_return stylesheet
+
+        dom = dom_fragment %(<link rel="stylesheet" href="//example.com/style.css">)
+        scanner = AssetScanner.new dom, normal_provider, external_provider
 
         expect(scanner.find_css).to eq([stylesheet])
       end
 
       it "ignores referenced print stylesheets" do
         dom = dom_fragment %(<link rel="stylesheet" href="/error.css" media="print">)
-        expect(provider).not_to receive(:find_stylesheet!)
+        expect(normal_provider).not_to receive(:find_stylesheet!)
 
-        scanner = AssetScanner.new dom, provider
+        scanner = AssetScanner.new dom, normal_provider, external_provider
 
         expect(scanner.find_css).to eq([])
       end
 
-      it "does not look for externally referenced stylesheets" do
+      it "does not look for externally referenced stylesheets from empty ProviderList" do
+        external_provider = ProviderList.empty
+
         dom = dom_fragment %(<link rel="stylesheet" href="//example.com/assets/style.css">)
-        expect(provider).not_to receive(:find_stylesheet!)
+        expect(external_provider).not_to receive(:find_stylesheet!)
 
-        scanner = AssetScanner.new dom, provider
+        scanner = AssetScanner.new dom, normal_provider, external_provider
 
         expect(scanner.find_css).to eq([])
       end
 
       it "does not look for ignored referenced stylesheets" do
         dom = dom_fragment %(<link rel="stylesheet" href="/error.css" data-roadie-ignore>)
-        expect(provider).not_to receive(:find_stylesheet!)
+        expect(normal_provider).not_to receive(:find_stylesheet!)
 
-        scanner = AssetScanner.new dom, provider
+        scanner = AssetScanner.new dom, normal_provider, external_provider
 
         expect(scanner.find_css).to eq([])
       end
@@ -103,7 +118,7 @@ module Roadie
           -->
         ]]></style>)
 
-        scanner = AssetScanner.new dom, provider
+        scanner = AssetScanner.new dom, normal_provider, external_provider
         stylesheet = scanner.find_css.first
 
         expect(stylesheet.to_s).to include("green")
@@ -119,7 +134,7 @@ module Roadie
           </script>
         HTML
 
-        scanner = AssetScanner.new dom, provider
+        scanner = AssetScanner.new dom, normal_provider, external_provider
         expect(scanner.find_css).to eq([])
       end
     end
@@ -133,15 +148,15 @@ module Roadie
               <style>span { color: green; }</style>
               <link rel="stylesheet" href="/some/url.css">
               <link rel="stylesheet" href="/error.css" media="print">
-              <link rel="stylesheet" href="/cool.css" data-roadie-ignore>
+              <link rel="stylesheet" href="/cool.css" data-roadie-ignore class="totally-ignored">
             </head>
             <body>
-              <style data-roadie-ignore>a { color: red; }</style>
+              <style data-roadie-ignore class="totally-ignored">a { color: red; }</style>
             </body>
           </html>
         HTML
-        provider = TestProvider.new "/some/url.css" => "body { color: green; }"
-        scanner = AssetScanner.new dom, provider
+        normal_provider = TestProvider.new "/some/url.css" => "body { color: green; }"
+        scanner = AssetScanner.new dom, normal_provider, external_provider
 
         stylesheets = scanner.extract_css
 
@@ -150,13 +165,52 @@ module Roadie
         expect(stylesheets[1].to_s).to include("body")
 
         expect(dom).to have_selector("html > head > title")
-        expect(dom).to have_selector("html > body > style[data-roadie-ignore]")
-        expect(dom).to have_selector("link[data-roadie-ignore]")
+        expect(dom).to have_selector("html > body > style.totally-ignored")
+        expect(dom).to have_selector("link.totally-ignored")
         expect(dom).to have_selector("link[media=print]")
 
         expect(dom).not_to have_selector("html > head > style")
         expect(dom).not_to have_selector("html > head > link[href='/some/url.css']")
       end
+
+      it "removes external references if provider is not empty" do
+        dom = dom_document <<-HTML
+          <html>
+            <head>
+              <link rel="stylesheet" href="//some/url.css">
+              <link rel="stylesheet" href="//other/url.css" data-roadie-ignore>
+            </head>
+          </html>
+        HTML
+        external_provider = ProviderList.wrap(NullProvider.new)
+        scanner = AssetScanner.new dom, normal_provider, external_provider
+
+        stylesheets = scanner.extract_css
+
+        expect(stylesheets).to have(1).stylesheets
+
+        expect(dom).to_not have_selector("link[href*=some]")
+        expect(dom).to have_selector("link[href*=other]")
+      end
+
+      it "removes the data-roadie-ignore markers" do
+        dom = dom_document <<-HTML
+          <html>
+            <head>
+              <link rel="stylesheet" href="/cool.css" data-roadie-ignore id="first">
+            </head>
+            <body>
+              <style data-roadie-ignore id="second">a { color: red; }</style>
+            </body>
+          </html>
+        HTML
+        scanner = AssetScanner.new dom, TestProvider.new, external_provider
+
+        scanner.extract_css
+
+        expect(dom.at_css("#first").attributes).to_not include("data-roadie-ignore")
+        expect(dom.at_css("#second").attributes).to_not include("data-roadie-ignore")
+      end
     end
   end
 end