rails_den 2.0.0

data/lib/rails_den/onebox/body_renderer.rb

@@ -0,0 +1,135 @@
+require "active_support/core_ext/string/output_safety"
+
+module RailsDen
+  module Onebox
+    class BodyRenderer
+      def self.call(post, profile_path_resolver: nil)
+        new(
+          post,
+          profile_path_resolver: profile_path_resolver
+        ).call
+      end
+
+      def initialize(post, profile_path_resolver: nil)
+        unless post.is_a?(RailsDen::Post)
+          raise ArgumentError,
+                "post must be a RailsDen::Post"
+        end
+
+        @post = post
+        @profile_path_resolver = profile_path_resolver
+      end
+
+      def call
+        return formatted_body unless RailsDen.config.onebox_enabled
+
+        occurrences =
+          Extractor.occurrences(
+            post.body
+          )
+
+        return formatted_body if occurrences.empty?
+
+        results =
+          resolved_results
+
+        return formatted_body if results.empty?
+
+        occurrences_by_line =
+          occurrences.index_by(
+            &:line_index
+          )
+
+        output =
+          ActiveSupport::SafeBuffer.new
+
+        buffer = +""
+
+        post.body.to_s
+            .each_line
+            .with_index do |line, line_index|
+          occurrence =
+            occurrences_by_line[
+              line_index
+            ]
+
+          result =
+            if occurrence
+              results[
+                occurrence.url
+              ]
+            end
+
+          rendered =
+            if result
+              Renderer.call(result)
+            end
+
+          if rendered &&
+             !rendered.to_s.empty?
+            append_formatted(
+              output,
+              buffer
+            )
+
+            buffer.clear
+
+            output.safe_concat(
+              rendered.to_s
+            )
+          else
+            buffer << line
+          end
+        end
+
+        append_formatted(
+          output,
+          buffer
+        )
+
+        output
+      end
+
+      private
+
+      attr_reader :post, :profile_path_resolver
+
+      def formatted_body
+        RailsDen::Mentions::Renderer.call(
+          source: post.body,
+          mentions: post.mentions,
+          profile_path_resolver: profile_path_resolver
+        )
+      end
+
+      def resolved_results
+        post.onebox_entries
+            .to_a
+            .each_with_object({}) do |entry, results|
+          result = entry.result
+
+          next unless result
+
+          results[
+            entry.source_url
+          ] = result
+        end
+      end
+
+      def append_formatted(output, source)
+        return if source.empty?
+
+        formatted =
+          RailsDen::Mentions::Renderer.call(
+            source: source,
+            mentions: post.mentions,
+            profile_path_resolver: profile_path_resolver
+          )
+
+        output.safe_concat(
+          formatted.to_s
+        )
+      end
+    end
+  end
+end

data/lib/rails_den/onebox/embed_policy.rb

@@ -0,0 +1,179 @@
+require "ipaddr"
+require "uri"
+
+module RailsDen
+  module Onebox
+    class EmbedPolicy
+      Rule = Struct.new(
+        :host,
+        :include_subdomains,
+        keyword_init: true
+      )
+
+      class << self
+        def register_host(
+          host,
+          include_subdomains: false
+        )
+          normalized_host =
+            normalize_registered_host(host)
+
+          rule = Rule.new(
+            host: normalized_host,
+            include_subdomains:
+              !!include_subdomains
+          )
+
+          mutex.synchronize do
+            rules.delete_if do |existing|
+              existing.host == rule.host &&
+                existing.include_subdomains ==
+                  rule.include_subdomains
+            end
+
+            rules << rule
+          end
+
+          rule
+        end
+
+        def allowed?(
+          url,
+          additional_hosts: []
+        )
+          uri = parse_embed_uri(url)
+
+          return false unless uri
+
+          host =
+            normalize_hostname(
+              uri.hostname
+            )
+
+          return false unless host
+
+          return true if registered_rules.any? do |rule|
+            host_matches?(
+              host,
+              rule
+            )
+          end
+
+          normalized_additional_hosts(
+            additional_hosts
+          ).include?(host)
+        end
+
+        def registered_rules
+          mutex.synchronize do
+            rules.dup.freeze
+          end
+        end
+
+        def reset!
+          mutex.synchronize do
+            @rules = []
+          end
+        end
+
+        private
+
+        def rules
+          @rules ||= []
+        end
+
+        def mutex
+          @mutex ||= Mutex.new
+        end
+
+        def parse_embed_uri(value)
+          candidate =
+            value.to_s.strip
+
+          return if candidate.empty?
+
+          uri = URI.parse(candidate)
+
+          return unless uri.scheme == "https"
+          return unless uri.hostname
+          return unless uri.port == 443
+          return if uri.userinfo
+
+          uri
+        rescue URI::InvalidURIError
+          nil
+        end
+
+        def normalize_registered_host(value)
+          host =
+            normalize_hostname(value)
+
+          if host.nil?
+            raise ArgumentError,
+                  "Onebox embed host is invalid"
+          end
+
+          if ip_address?(host)
+            raise ArgumentError,
+                  "Onebox embed host cannot be an IP address"
+          end
+
+          if host == "localhost" ||
+             !host.include?(".")
+            raise ArgumentError,
+                  "Onebox embed host must be a public hostname"
+          end
+
+          host
+        end
+
+        def normalized_additional_hosts(values)
+          Array(values)
+            .filter_map do |value|
+              begin
+                normalize_registered_host(
+                  value
+                )
+              rescue ArgumentError
+                nil
+              end
+            end
+            .uniq
+        end
+
+        def normalize_hostname(value)
+          host =
+            value.to_s
+                 .strip
+                 .downcase
+                 .delete_suffix(".")
+
+          return if host.empty?
+          return if host.match?(/\s/)
+
+          host
+        end
+
+        def ip_address?(host)
+          IPAddr.new(host)
+
+          true
+        rescue IPAddr::InvalidAddressError
+          false
+        end
+
+        def host_matches?(
+          host,
+          rule
+        )
+          return true if host == rule.host
+          return false unless rule.include_subdomains
+
+          host.end_with?(
+            ".#{rule.host}"
+          )
+        end
+      end
+    end
+  end
+end

data/lib/rails_den/onebox/errors.rb

@@ -0,0 +1,24 @@
+module RailsDen
+  module Onebox
+    class Error < StandardError
+    end
+
+    class UnsafeUrlError < Error
+    end
+
+    class ResolutionError < Error
+    end
+
+    class FetchError < Error
+    end
+
+    class ResponseTooLargeError < Error
+    end
+
+    class TooManyRedirectsError < Error
+    end
+
+    class InsecureRedirectError < Error
+    end
+  end
+end

data/lib/rails_den/onebox/extractor.rb

@@ -0,0 +1,130 @@
+module RailsDen
+  module Onebox
+    class Extractor
+      Occurrence =
+        Struct.new(
+          :url,
+          :line_index,
+          keyword_init: true
+        )
+
+      FENCE_PATTERN =
+        /\A {0,3}(`{3,}|~{3,})/
+
+      def self.call(text)
+        new(text).call
+      end
+
+      def self.occurrences(text)
+        new(text).occurrences
+      end
+
+      def initialize(text)
+        @text = text.to_s
+      end
+
+      def call
+        seen = {}
+
+        occurrences.each_with_object([]) do |occurrence, urls|
+          next if seen[occurrence.url]
+
+          seen[occurrence.url] = true
+          urls << occurrence.url
+        end
+      end
+
+      def occurrences
+        results = []
+        fence = nil
+
+        @text.each_line.with_index do |raw_line, line_index|
+          line = raw_line.chomp
+
+          if fence
+            fence = nil if closing_fence?(
+              line,
+              fence
+            )
+
+            next
+          end
+
+          opening = opening_fence(line)
+
+          if opening
+            fence = opening
+            next
+          end
+
+          next if indented_code?(line)
+
+          candidate = standalone_url(line)
+
+          next unless candidate
+          next unless Url.web?(candidate)
+
+          results <<
+            Occurrence.new(
+              url: candidate,
+              line_index: line_index
+            )
+        end
+
+        results
+      end
+
+      private
+
+      def opening_fence(line)
+        match = line.match(FENCE_PATTERN)
+
+        return unless match
+
+        marker = match[1]
+
+        {
+          character: marker[0],
+          length: marker.length
+        }
+      end
+
+      def closing_fence?(line, fence)
+        character =
+          Regexp.escape(
+            fence.fetch(:character)
+          )
+
+        minimum_length =
+          fence.fetch(:length)
+
+        line.match?(
+          /\A {0,3}#{character}{#{minimum_length},}\s*\z/
+        )
+      end
+
+      def indented_code?(line)
+        line.start_with?("    ") ||
+          line.start_with?("\t")
+      end
+
+      def standalone_url(line)
+        stripped = line.strip
+
+        return if stripped.empty?
+
+        if stripped.start_with?("<") &&
+           stripped.end_with?(">")
+          candidate =
+            stripped[1...-1].to_s.strip
+
+          return candidate unless candidate.empty?
+        end
+
+        return if stripped.match?(/\s/)
+
+        stripped
+      end
+    end
+  end
+end

data/lib/rails_den/onebox/fetcher.rb

@@ -0,0 +1,199 @@
+require "uri"
+
+module RailsDen
+  module Onebox
+    class Fetcher
+      Response = Struct.new(
+        :original_url,
+        :url,
+        :status,
+        :headers,
+        :body,
+        keyword_init: true
+      )
+
+      def self.call(url, **options)
+        new(**options).call(url)
+      end
+
+      def initialize(
+        address_guard: nil,
+        transport: nil,
+        allowed_ports: nil,
+        open_timeout: nil,
+        read_timeout: nil,
+        max_response_bytes: nil,
+        max_redirects: nil,
+        user_agent: nil
+      )
+        @address_guard =
+          address_guard ||
+          AddressGuard.new
+
+        @transport =
+          transport ||
+          HttpTransport.new
+
+        @allowed_ports =
+          Array(
+            allowed_ports ||
+            RailsDen.config.onebox_allowed_ports
+          ).map(&:to_i).freeze
+
+        @open_timeout =
+          open_timeout ||
+          RailsDen.config.onebox_open_timeout
+
+        @read_timeout =
+          read_timeout ||
+          RailsDen.config.onebox_read_timeout
+
+        @max_response_bytes =
+          max_response_bytes ||
+          RailsDen.config.onebox_max_response_bytes
+
+        @max_redirects =
+          if max_redirects.nil?
+            RailsDen.config.onebox_max_redirects
+          else
+            max_redirects
+          end
+
+        @user_agent =
+          user_agent ||
+          RailsDen.config.onebox_user_agent
+      end
+
+      def call(url)
+        original_uri = validate_uri(url)
+        current_uri = original_uri
+        redirects = 0
+
+        loop do
+          addresses =
+            @address_guard.resolve(
+              current_uri.hostname
+            )
+
+          response =
+            fetch_from_addresses(
+              current_uri,
+              addresses
+            )
+
+          unless redirect?(response)
+            return Response.new(
+              original_url: original_uri.to_s,
+              url: current_uri.to_s,
+              status: response.status,
+              headers: response.headers,
+              body: response.body
+            )
+          end
+
+          if redirects >= @max_redirects
+            raise TooManyRedirectsError,
+                  "Onebox redirect limit exceeded"
+          end
+
+          current_uri =
+            redirected_uri(
+              current_uri,
+              response
+            )
+
+          redirects += 1
+        end
+      end
+
+      private
+
+      def fetch_from_addresses(uri, addresses)
+        last_error = nil
+
+        addresses.each do |ip_address|
+          begin
+            return @transport.get(
+              uri,
+              ip_address: ip_address,
+              open_timeout: @open_timeout,
+              read_timeout: @read_timeout,
+              max_bytes: @max_response_bytes,
+              user_agent: @user_agent
+            )
+          rescue FetchError => error
+            last_error = error
+          end
+        end
+
+        raise last_error if last_error
+
+        raise FetchError,
+              "Onebox request failed for all resolved addresses"
+      end
+
+      def validate_uri(value)
+        candidate = value.to_s.strip
+
+        unless Url.web?(candidate)
+          raise UnsafeUrlError,
+                "Onebox URL must use HTTP or HTTPS"
+        end
+
+        uri = URI.parse(candidate)
+
+        unless uri.hostname
+          raise UnsafeUrlError,
+                "Onebox URL has no hostname"
+        end
+
+        unless @allowed_ports.include?(uri.port)
+          raise UnsafeUrlError,
+                "Onebox URL uses a disallowed port"
+        end
+
+        uri
+      rescue URI::InvalidURIError
+        raise UnsafeUrlError,
+              "Onebox URL is invalid"
+      end
+
+      def redirect?(response)
+        HttpTransport::REDIRECT_STATUSES.include?(
+          response.status
+        )
+      end
+
+      def redirected_uri(current_uri, response)
+        location =
+          response.headers[
+            "location"
+          ].to_s.strip
+
+        if location.empty?
+          raise UnsafeUrlError,
+                "Onebox redirect has no location"
+        end
+
+        redirected =
+          URI.join(
+            current_uri.to_s,
+            location
+          )
+
+        if current_uri.scheme == "https" &&
+           redirected.scheme == "http"
+          raise InsecureRedirectError,
+                "Onebox HTTPS redirect cannot downgrade to HTTP"
+        end
+
+        validate_uri(
+          redirected.to_s
+        )
+      rescue URI::InvalidURIError
+        raise UnsafeUrlError,
+              "Onebox redirect URL is invalid"
+      end
+    end
+  end
+end