rails_den 2.0.0

data/lib/rails_den/onebox/http_transport.rb

@@ -0,0 +1,164 @@
+require "net/http"
+require "openssl"
+
+module RailsDen
+  module Onebox
+    class HttpTransport
+      REDIRECT_STATUSES = [
+        301,
+        302,
+        303,
+        307,
+        308
+      ].freeze
+
+      RawResponse = Struct.new(
+        :status,
+        :headers,
+        :body,
+        keyword_init: true
+      )
+
+      def initialize(http_factory: nil)
+        @http_factory =
+          http_factory ||
+          lambda do |host, port|
+            Net::HTTP.new(host, port)
+          end
+      end
+
+      def get(
+        uri,
+        ip_address:,
+        open_timeout:,
+        read_timeout:,
+        max_bytes:,
+        user_agent:
+      )
+        http = @http_factory.call(
+          uri.hostname,
+          uri.port
+        )
+
+        configure_http(
+          http,
+          uri,
+          ip_address,
+          open_timeout,
+          read_timeout
+        )
+
+        request =
+          Net::HTTP::Get.new(
+            uri.request_uri
+          )
+
+        request["User-Agent"] = user_agent
+
+        request["Accept"] =
+          "text/html, application/xhtml+xml, application/json;q=0.9, */*;q=0.1"
+
+        request["Accept-Encoding"] = "identity"
+        request["Connection"] = "close"
+
+        result = nil
+
+        http.start do |connection|
+          connection.request(request) do |response|
+            status = response.code.to_i
+            headers = response_headers(response)
+
+            body =
+              if REDIRECT_STATUSES.include?(status)
+                ""
+              else
+                read_body(
+                  response,
+                  max_bytes
+                )
+              end
+
+            result = RawResponse.new(
+              status: status,
+              headers: headers,
+              body: body
+            )
+          end
+        end
+
+        result
+      rescue RailsDen::Onebox::Error
+        raise
+      rescue StandardError => error
+        raise FetchError,
+              "Onebox request failed: #{error.class}: #{error.message}"
+      end
+
+      private
+
+      def configure_http(
+        http,
+        uri,
+        ip_address,
+        open_timeout,
+        read_timeout
+      )
+        http.ipaddr = ip_address
+        http.open_timeout = open_timeout
+        http.read_timeout = read_timeout
+
+        if http.respond_to?(:write_timeout=)
+          http.write_timeout = read_timeout
+        end
+
+        if http.respond_to?(:max_retries=)
+          http.max_retries = 0
+        end
+
+        return unless uri.scheme == "https"
+
+        http.use_ssl = true
+        http.verify_mode =
+          OpenSSL::SSL::VERIFY_PEER
+      end
+
+      def response_headers(response)
+        response.each_header.each_with_object({}) do |(name, value), headers|
+          headers[name.downcase] = value
+        end
+      end
+
+      def read_body(response, max_bytes)
+        content_length =
+          integer_header(
+            response["content-length"]
+          )
+
+        if content_length &&
+           content_length > max_bytes
+          raise ResponseTooLargeError,
+                "Onebox response exceeds maximum size"
+        end
+
+        body = +"".b
+
+        response.read_body do |chunk|
+          if body.bytesize + chunk.bytesize > max_bytes
+            raise ResponseTooLargeError,
+                  "Onebox response exceeds maximum size"
+          end
+
+          body << chunk
+        end
+
+        body
+      end
+
+      def integer_header(value)
+        return if value.nil?
+
+        Integer(value, exception: false)
+      end
+    end
+  end
+end

data/lib/rails_den/onebox/metadata.rb

@@ -0,0 +1,343 @@
+require "nokogiri"
+require "uri"
+
+module RailsDen
+  module Onebox
+    class Metadata
+      TITLE_LIMIT = 300
+      DESCRIPTION_LIMIT = 2_000
+
+      attr_reader :page_url
+
+      def self.call(html, page_url:)
+        new(
+          html,
+          page_url: page_url
+        ).to_h
+      end
+
+      def initialize(html, page_url:)
+        @html = html.to_s
+        @page_url = page_url.to_s
+        @document =
+          Nokogiri::HTML5.parse(
+            @html
+          )
+      end
+
+      def to_h
+        {
+          title: title,
+          description: description,
+          canonical_url: canonical_url,
+          site_name: site_name,
+          type: object_type,
+          image_url: image_url,
+          image_alt: image_alt,
+          image_width: image_width,
+          image_height: image_height,
+          author_name: author_name,
+          published_at: published_at,
+          twitter_card: twitter_card,
+          raw: raw_metadata
+        }
+      end
+
+      private
+
+      def title
+        truncate(
+          first_present(
+            property("og:title"),
+            name("twitter:title"),
+            html_title
+          ),
+          TITLE_LIMIT
+        )
+      end
+
+      def description
+        truncate(
+          first_present(
+            property("og:description"),
+            name("twitter:description"),
+            name("description")
+          ),
+          DESCRIPTION_LIMIT
+        )
+      end
+
+      def canonical_url
+        resolve_web_url(
+          first_present(
+            property("og:url"),
+            canonical_link,
+            page_url
+          )
+        ) || page_url
+      end
+
+      def site_name
+        normalize_text(
+          first_present(
+            property("og:site_name"),
+            name("application-name")
+          )
+        )
+      end
+
+      def object_type
+        normalize_text(
+          property("og:type")
+        )
+      end
+
+      def image_url
+        value =
+          first_present(
+            property("og:image:secure_url"),
+            property("og:image"),
+            name("twitter:image"),
+            name("twitter:image:src")
+          )
+
+        resolve_web_url(value)
+      end
+
+      def image_alt
+        normalize_text(
+          first_present(
+            property("og:image:alt"),
+            name("twitter:image:alt")
+          )
+        )
+      end
+
+      def image_width
+        positive_integer(
+          property("og:image:width")
+        )
+      end
+
+      def image_height
+        positive_integer(
+          property("og:image:height")
+        )
+      end
+
+      def author_name
+        normalize_text(
+          first_present(
+            property("article:author"),
+            name("author")
+          )
+        )
+      end
+
+      def published_at
+        normalize_text(
+          first_present(
+            property("article:published_time"),
+            itemprop("datePublished")
+          )
+        )
+      end
+
+      def twitter_card
+        normalize_text(
+          name("twitter:card")
+        )
+      end
+
+      def raw_metadata
+        {
+          open_graph: open_graph_metadata,
+          twitter: twitter_metadata
+        }
+      end
+
+      def open_graph_metadata
+        metadata_for_attribute(
+          "property",
+          /\A(?:og|article|profile|video|music):/
+        )
+      end
+
+      def twitter_metadata
+        metadata_for_attribute(
+          "name",
+          /\Atwitter:/
+        )
+      end
+
+      def metadata_for_attribute(
+        attribute,
+        pattern
+      )
+        result = {}
+
+        @document.css(
+          "meta[#{attribute}]"
+        ).each do |node|
+          key =
+            node[
+              attribute
+            ].to_s.strip.downcase
+
+          next unless key.match?(pattern)
+          next if result.key?(key)
+
+          value =
+            normalize_text(
+              node["content"]
+            )
+
+          next if value.nil?
+
+          result[key] = value
+        end
+
+        result.freeze
+      end
+
+      def property(key)
+        meta_content(
+          "property",
+          key
+        )
+      end
+
+      def name(key)
+        meta_content(
+          "name",
+          key
+        )
+      end
+
+      def itemprop(key)
+        meta_content(
+          "itemprop",
+          key
+        )
+      end
+
+      def meta_content(
+        attribute,
+        key
+      )
+        target = key.downcase
+
+        node =
+          @document.css(
+            "meta[#{attribute}]"
+          ).find do |meta|
+            meta[
+              attribute
+            ].to_s.strip.downcase == target
+          end
+
+        normalize_text(
+          node&.[]("content")
+        )
+      end
+
+      def html_title
+        normalize_text(
+          @document.at_css(
+            "title"
+          )&.text
+        )
+      end
+
+      def canonical_link
+        node =
+          @document.css(
+            "link[href]"
+          ).find do |link|
+            link[
+              "rel"
+            ].to_s
+             .split(/\s+/)
+             .any? do |rel|
+               rel.casecmp?(
+                 "canonical"
+               )
+             end
+          end
+
+        normalize_text(
+          node&.[]("href")
+        )
+      end
+
+      def resolve_web_url(value)
+        candidate =
+          normalize_text(value)
+
+        return if candidate.nil?
+
+        resolved =
+          URI.join(
+            page_url,
+            candidate
+          )
+
+        return unless Url.web?(
+          resolved.to_s
+        )
+
+        resolved.to_s
+      rescue URI::InvalidURIError
+        nil
+      end
+
+      def positive_integer(value)
+        integer =
+          Integer(
+            value,
+            exception: false
+          )
+
+        return unless integer
+        return unless integer.positive?
+
+        integer
+      end
+
+      def first_present(*values)
+        values.find do |value|
+          !normalize_text(value).nil?
+        end
+      end
+
+      def normalize_text(value)
+        string =
+          value.to_s
+               .encode(
+                 Encoding::UTF_8,
+                 invalid: :replace,
+                 undef: :replace,
+                 replace: ""
+               )
+               .gsub(/\s+/, " ")
+               .strip
+
+        return if string.empty?
+
+        string
+      end
+
+      def truncate(value, limit)
+        string =
+          normalize_text(value)
+
+        return unless string
+
+        string[
+          0,
+          limit
+        ]
+      end
+    end
+  end
+end

data/lib/rails_den/onebox/oembed/client.rb

@@ -0,0 +1,46 @@
+module RailsDen
+  module Onebox
+    module OEmbed
+      class Client
+        def self.call(
+          endpoint,
+          fetcher: Fetcher
+        )
+          new(
+            fetcher: fetcher
+          ).call(endpoint)
+        end
+
+        def initialize(fetcher: Fetcher)
+          @fetcher = fetcher
+        end
+
+        def call(endpoint)
+          unless endpoint.respond_to?(:url) &&
+                 endpoint.respond_to?(:format)
+            raise ArgumentError,
+                  "oEmbed endpoint is required"
+          end
+
+          response =
+            @fetcher.call(
+              endpoint.url
+            )
+
+          unless response.status.between?(
+            200,
+            299
+          )
+            raise Error,
+                  "oEmbed endpoint returned HTTP #{response.status}"
+          end
+
+          Response.parse(
+            response.body,
+            format: endpoint.format
+          )
+        end
+      end
+    end
+  end
+end