link_preview 0.2.7

data/lib/link_preview/content.rb

@@ -0,0 +1,350 @@
+# Copyright (c) 2014, VMware, Inc. All Rights Reserved.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights to
+# use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
+# of the Software, and to permit persons to whom the Software is furnished to do
+# so, subject to the following conditions:
+# 
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+# 
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+
+require 'link_preview/uri'
+require 'link_preview/parser'
+require 'link_preview/http_crawler'
+require 'link_preview/null_crawler'
+
+require 'active_support/core_ext/object'
+
+module LinkPreview
+  class Content
+    PROPERTIES = [
+      :title,
+      :description,
+      :site_name,
+      :site_url,
+      :image_url,
+      :image_data,
+      :image_content_type,
+      :image_file_name,
+      :content_url,
+      :content_type,
+      :content_width,
+      :content_height ]
+
+    SOURCES = [:initial, :image, :oembed, :opengraph, :html]
+
+    SOURCE_PROPERTIES_TABLE =
+      {
+        :oembed =>
+        {
+          :site_name => :provider_name,
+          :site_url => :provider_url,
+          :image_url => :thumbnail_url
+        },
+        :opengraph =>
+        {
+          :image_url => [:image_secure_url, :image, :image_url],
+          :content_url => [:video_secure_url, :video, :video_url],
+          :content_type => :video_type,
+          :content_width => :video_width,
+          :content_height => :video_height
+        }
+      }
+
+    PROPERTIES_SOURCE_TABLE =
+      Hash.new { |h,k| h[k] = {} }.tap do |reverse_property_table|
+        SOURCE_PROPERTIES_TABLE.each do |source, table|
+          table.invert.each_pair do |keys, val|
+            Array.wrap(keys).each do |key|
+              reverse_property_table[source][key] = val
+            end
+          end
+        end
+      end
+
+    def initialize(config, content_uri, options = {}, sources = {})
+      @config = config
+      @content_uri = content_uri
+      @options = options
+      @sources = Hash.new { |h,k| h[k] = {} }
+      crawler.enqueue!(@content_uri)
+
+      add_source_properties!(sources)
+    end
+
+    # @return [String] permalink URL of resource
+    def url
+      extract(:url) || @content_uri
+    end
+
+    PROPERTIES.each do |property|
+      define_method(property) do
+        extract(property)
+      end
+    end
+
+    # @return [Boolean] true of at least related content URI has been successfully fetched
+    def found?
+      extract_all
+      crawler.success?
+    end
+
+    # @return [Boolean] true of at least one content property is present
+    def empty?
+      extract_all
+      SOURCES.none? do |source|
+        @sources[source].any?(&:present?)
+      end
+    end
+
+    def sources
+      @sources
+    end
+
+    def as_oembed
+      if content_type == 'application/x-shockwave-flash'
+        @sources[:oembed].reverse_merge(as_oembed_video)
+      else
+        @sources[:oembed].reverse_merge(as_oembed_link)
+      end
+    end
+
+    protected
+
+    def crawler
+      @crawler ||= @options.fetch(:allow_requests, true) ?
+        LinkPreview::HTTPCrawler.new(@config, @options) :
+        LinkPreview::NullCrawler.new(@config, @options)
+    end
+
+    def parser
+      @parser ||= LinkPreview::Parser.new(@config, @options)
+    end
+
+    def parsed_url
+      LinkPreview::URI.parse(url, @options)
+    end
+
+    def default_property(property)
+      if respond_to?("default_#{property}", true)
+        send("default_#{property}")
+      end
+    end
+
+    # called via default_property
+    def default_title
+      parsed_url.for_display.to_s
+    end
+
+    # called via default_property
+    def default_site_name
+      parsed_url.host
+    end
+
+    # called via default_property
+    def default_site_url
+      if parsed_url.scheme && parsed_url.host
+        "#{parsed_url.scheme}://#{parsed_url.host}"
+      end
+    end
+
+    def normalize_property(property, value)
+      if respond_to?("normalize_#{property}", true)
+        send("normalize_#{property}", value)
+      else
+        normalize_generic(property, value)
+      end
+    end
+
+    def normalize_generic(property, value)
+      case value
+      when String
+        strip_html(value.strip)
+      when Array
+        value.compact.map { |elem| normalize_property(property, elem ) }
+      else
+        value
+      end
+    end
+
+    # called via normalize_property
+    def normalize_image_url(partial_image_url)
+      return unless partial_image_url
+      parsed_partial_image_url = LinkPreview::URI.parse(partial_image_url, @options)
+      parsed_absolute_image_url = parsed_partial_image_url.to_absolute(@content_uri)
+      parsed_absolute_image_url.to_s.tap do |absolute_image_url|
+        crawler.enqueue!(absolute_image_url, :image)
+      end
+    end
+
+    # called via normalize_property
+    def normalize_url(partial_url)
+      return unless partial_url
+      partial_unencoded_url = LinkPreview::URI.unescape(partial_url)
+      parsed_partial_url = LinkPreview::URI.parse(partial_unencoded_url, @options)
+      parsed_absolute_url = parsed_partial_url.to_absolute(@content_uri)
+      crawler.enqueue!(parsed_absolute_url, :html)
+      parsed_absolute_url.for_display.to_s
+    end
+
+    # called via normalize_property
+    def normalize_content_url(content_url)
+      return unless content_url
+      LinkPreview::URI.safe_escape(content_url).to_s
+    end
+
+    # called via normalize_property
+    def normalize_title(title)
+      CGI.unescapeHTML(title)
+    end
+
+    # called via normalize_property
+    def normalize_html(html)
+      html
+    end
+
+    def get_property(property)
+      SOURCES.map do |source|
+        @sources[source][property_alias(source, property)]
+      end.compact.first || default_property(property)
+    end
+
+    def has_property?(property)
+      SOURCES.map do |source|
+        @sources[source][property_alias(source, property)]
+      end.any?(&:present?)
+    end
+
+    def property_alias(source, property)
+      property_aliases(source,property).detect { |property| @sources[source].has_key?(property) }
+    end
+
+    def property_aliases(source, property)
+      Array.wrap(SOURCE_PROPERTIES_TABLE.fetch(source, {}).fetch(property, property))
+    end
+
+    def property_unalias(source, property)
+      PROPERTIES_SOURCE_TABLE.fetch(source, {}).fetch(property, property)
+    end
+
+    def property_source_priority(property)
+      case property
+      when :description
+        [:html, :oembed, :default]
+      when :image_data, :image_content_type, :image_file_name
+        [:image, :oembed, :default]
+      else
+        [:oembed, :html, :image, :default]
+      end
+    end
+
+    def add_source_properties!(sources)
+      sources.symbolize_keys!
+      sources.reject!{ |_, properties| properties.empty? }
+      sources.select! { |source,_| SOURCES.include?(source) }
+      sources.each do |source, properties|
+        properties.symbolize_keys!
+        properties.reject!{ |_, value| value.blank? }
+        properties.each do |property, value|
+          next if @sources[source][property]
+          @sources[source][property] = normalize_property(property_unalias(source, property), value)
+        end
+      end
+      parser.discovered_uris.each do |uri|
+        crawler.enqueue!(uri)
+      end
+    end
+
+    def extract(property)
+      while !crawler.finished? do
+        break if has_property?(property)
+        data = crawler.dequeue!(property_source_priority(property))
+        properties = parser.parse(data)
+        add_source_properties!(properties)
+      end
+      get_property(property)
+    end
+
+    def extract_all
+      PROPERTIES.each do |property|
+        send(property)
+      end
+    end
+
+    # FIXME this is expensive
+    def strip_html(value)
+      Nokogiri::HTML(value).xpath('//text()').remove.to_s
+    end
+
+    def as_oembed_link
+      {
+        :version         => '1.0',
+        :provider_name   => site_name,
+        :provider_url    => site_url,
+        :title           => title,
+        :description     => description,
+        :type            => 'link',
+        :thumbnail_url   => image_url
+      }.reject { |_,v| v.nil? }
+    end
+
+    def as_oembed_video
+      as_oembed_link.merge({
+          :type            => 'video',
+          :html            => content_html,
+          :width           => content_width_scaled.to_i,
+          :height          => content_height_scaled.to_i})
+    end
+
+    def content_html
+      return nil unless content_url.present?
+
+      <<-EOF.strip.gsub(/\s+/, ' ').gsub(/>\s+</, '><')
+          <object width="#{content_width_scaled}" height="#{content_height_scaled}">
+            <param name="movie" value="#{content_url}"></param>
+            <param name="allowScriptAccess" value="always"></param>
+            <param name="allowFullScreen" value="true"></param>
+            <embed src="#{content_url}"
+                   type="#{content_type}"
+                   allowscriptaccess="always"
+                   allowfullscreen="true"
+                   width="#{content_width_scaled}" height="#{content_height_scaled}"></embed>
+          </object>
+      EOF
+    end
+
+    def content_width_scaled
+      # Width takes precedence over height
+      if @options[:width].to_i > 0
+        @options[:width]
+      elsif @options[:height].to_i > 0 && content_height.to_i > 0
+        # Compute scaled width using the ratio of requested height to actual height, round up to prevent truncation
+        (((@options[:height].to_i * 1.0) / (content_height.to_i * 1.0)) * content_width.to_i).ceil
+      else
+        content_width.to_i
+      end
+    end
+
+    def content_height_scaled
+      # Width takes precedence over height
+      if @options[:width].to_i > 0 && content_width.to_i > 0
+        # Compute scaled height using the ratio of requested width to actual width, round up to prevent truncation
+        (((@options[:width].to_i * 1.0) / (content_width.to_i * 1.0)) * content_height.to_i).ceil
+      elsif @options[:height].to_i > 0
+        @options[:height]
+      else
+        content_height.to_i
+      end
+    end
+  end
+end

data/lib/link_preview/http_client.rb

@@ -0,0 +1,91 @@
+# Copyright (c) 2014, VMware, Inc. All Rights Reserved.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights to
+# use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
+# of the Software, and to permit persons to whom the Software is furnished to do
+# so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+
+require 'faraday'
+require 'faraday/follow_redirects'
+
+module LinkPreview
+  class ExtraEnv < Faraday::Middleware
+    class << self
+      attr_accessor :extra
+    end
+
+    def call(env)
+      env[:link_preview] = self.class.extra || {}
+      @app.call(env)
+    ensure
+      env[:link_preview] = nil
+    end
+  end
+
+  class NormalizeURI < Faraday::Middleware
+    def call(env)
+      env[:url] = env[:url].normalize
+      @app.call(env)
+    end
+  end
+
+  class ForceUTF8Body < Faraday::Middleware
+    def force_utf8_body!(env)
+      return if env[:body].encoding == Encoding::UTF_8 && env[:body].valid_encoding?
+      return unless env[:response_headers][:content_type] =~ /text/
+      env[:body].encode!('UTF-8', 'binary', invalid: :replace, undef: :replace, replace: '')
+      unless env[:body].valid_encoding?
+        # cleanse untrusted invalid bytes with a double transcode as suggested here:
+        # http://stackoverflow.com/questions/2982677/ruby-1-9-invalid-byte-sequence-in-utf-8
+        env[:body].encode!('UTF-16', 'binary', invalid: :replace, undef: :replace, replace: '')
+        env[:body].encode!('UTF-8', 'binary', invalid: :replace, undef: :replace, replace: '')
+      end
+    end
+
+    def call(env)
+      @app.call(env).on_complete do |env|
+        force_utf8_body!(env)
+      end
+    end
+  end
+
+  class HTTPClient
+    extend Forwardable
+
+    def initialize(config)
+      @config = config
+    end
+
+    def_delegator :faraday_connection, :get
+
+    private
+
+    def faraday_connection
+      @faraday_connection ||= Faraday.new do |builder|
+        builder.options[:timeout] = @config.timeout
+        builder.options[:open_timeout] = @config.open_timeout
+
+        builder.use ExtraEnv
+        builder.use Faraday::FollowRedirects, limit: @config.max_redirects if @config.follow_redirects
+        builder.use NormalizeURI
+        builder.use ForceUTF8Body
+        @config.middleware.each { |middleware| builder.use middleware }
+
+        builder.use @config.http_adapter
+      end
+    end
+  end
+end

data/lib/link_preview/http_crawler.rb

@@ -0,0 +1,114 @@
+# Copyright (c) 2014, VMware, Inc. All Rights Reserved.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights to
+# use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
+# of the Software, and to permit persons to whom the Software is furnished to do
+# so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+
+require 'link_preview'
+require 'link_preview/uri'
+
+module LinkPreview
+  module ResponseWithURL
+    attr_accessor :url
+  end
+
+  class HTTPCrawler
+    def initialize(config, options = {})
+      @config = config
+      @options = options
+      @status = {}
+      @queue = Hash.new { |h,k| h[k] = [] }
+    end
+
+    # @param [String] URI of content to crawl
+    def enqueue!(uri, priority = :default)
+      return if full?
+      return unless uri
+      parsed_uri = LinkPreview::URI.parse(uri, @options)
+
+      if oembed_uri = parsed_uri.as_oembed_uri
+        enqueue_uri(oembed_uri, :oembed)
+      end
+
+      if content_uri = parsed_uri.as_content_uri
+        enqueue_uri(content_uri, priority)
+      end
+    end
+
+    # @return [Hash] latest normalized content discovered by crawling
+    def dequeue!(priority_order = [])
+      return if finished?
+      uri = dequeue_by_priority(priority_order)
+      with_extra_env do
+        @config.http_client.get(uri).tap do |response|
+          response.extend ResponseWithURL
+          response.url = uri
+          @status[uri] = response.status.to_i
+        end
+      end
+    rescue => e
+      @status[uri] ||= 500
+      @config.error_handler.call(e)
+      Faraday::Response.new
+    end
+
+    # @return [Boolean] true if any content discovered thus far has been successfully fetched
+    def success?
+      @status.any? { |_, status| status == 200 }
+    end
+
+    # @return [Boolean] true if all known discovered content has been crawled
+    def finished?
+      @queue.values.flatten.empty?
+    end
+
+    # @return [Boolean] true crawler is at capacity
+    def full?
+      @queue.values.flatten.size > @config.max_requests
+    end
+
+    private
+
+    def dequeue_by_priority(priority_order)
+      priority = priority_order.detect { |priority| @queue[priority].any? }
+      priority ||= @queue.keys.detect { |priority| @queue[priority].any? }
+      @queue[priority].shift
+    end
+
+    def enqueue_uri(parsed_uri, priority = :default)
+      uri = parsed_uri.to_s
+      if !(processed?(uri) || enqueued?(uri))
+        @queue[priority] << uri
+      end
+    end
+
+    def processed?(uri)
+      @status.has_key?(uri)
+    end
+
+    def enqueued?(uri)
+      @queue.values.flatten.uniq.include?(uri)
+    end
+
+    def with_extra_env(&block)
+      LinkPreview::ExtraEnv.extra = @options
+      yield
+    ensure
+      LinkPreview::ExtraEnv.extra = nil
+    end
+  end
+end

data/lib/link_preview/null_crawler.rb

@@ -0,0 +1,46 @@
+# Copyright (c) 2014, VMware, Inc. All Rights Reserved.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights to
+# use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
+# of the Software, and to permit persons to whom the Software is furnished to do
+# so, subject to the following conditions:
+# 
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+# 
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+
+require 'link_preview'
+
+module LinkPreview
+  class NullCrawler
+    def initialize(config, options = {})
+    end
+
+    def enqueue!(uri, priority = :default)
+    end
+
+    def dequeue!(priority_order = [])
+    end
+
+    def success?
+      true
+    end
+
+    def finished?
+      true
+    end
+
+    def full?
+      false
+    end
+  end
+end