brand_logo 1.0.0 → 2.0.0

data/lib/brand_logo/dimensions.rb

@@ -0,0 +1,79 @@
+# frozen_string_literal: true
+# typed: strict
+
+module BrandLogo
+  Dimensions = Data.define(:width, :height) do
+    extend T::Sig
+
+    sig { params(width: T.nilable(Integer), height: T.nilable(Integer)).void }
+    def initialize(width: nil, height: nil)
+      super
+    end
+
+    sig { params(value: T.untyped).returns(Dimensions) }
+    def self.parse(value)
+      return new unless value.is_a?(String)
+
+      sizes = value.split.filter_map do |token|
+        match = /\A(\d+)x(\d+)\z/i.match(token)
+        new(width: match[1].to_i, height: match[2].to_i) if match
+      end
+      sizes.select(&:known?).max_by(&:area) || new
+    end
+
+    sig { params(value: T.untyped).returns(T.nilable(Dimensions)) }
+    def self.coerce(value)
+      case value
+      when nil, Dimensions then value
+      when Integer then new(width: value, height: value)
+      when ::Hash then new(width: value.fetch(:width), height: value.fetch(:height))
+      else raise ValidationError, "Invalid dimensions: #{value.inspect}"
+      end
+    end
+
+    sig { returns(T.nilable([Integer, Integer])) }
+    def to_pair
+      w = width
+      h = height
+      [w, h] if w&.positive? && h&.positive?
+    end
+
+    sig { returns(T::Boolean) }
+    def known?
+      !to_pair.nil?
+    end
+
+    sig { returns(Integer) }
+    def area
+      w, h = to_pair
+      w && h ? w * h : 0
+    end
+
+    sig { returns(Integer) }
+    def side
+      to_pair&.min || 0
+    end
+
+    sig { params(other: Dimensions).returns(T::Boolean) }
+    def at_least?(other)
+      compare(other) { |mine, theirs| mine >= theirs }
+    end
+
+    sig { params(other: Dimensions).returns(T::Boolean) }
+    def at_most?(other)
+      compare(other) { |mine, theirs| mine <= theirs }
+    end
+
+    private
+
+    # Unknown dimensions on either side never fail a comparison.
+    sig { params(other: Dimensions, block: T.proc.params(mine: Integer, theirs: Integer).returns(T::Boolean)).returns(T::Boolean) }
+    def compare(other, &block)
+      mine = to_pair
+      theirs = other.to_pair
+      return true unless mine && theirs
+
+      mine.zip(theirs).all? { |m, t| yield(m, T.must(t)) }
+    end
+  end
+end

data/lib/brand_logo/domain.rb

@@ -0,0 +1,64 @@
+# frozen_string_literal: true
+# typed: strict
+
+require 'uri'
+
+module BrandLogo
+  module Domain
+    extend T::Sig
+
+    MAX_LENGTH = 253
+    LABEL = T.let(/\A[a-z0-9]([a-z0-9-]{0,61}[a-z0-9])?\z/, Regexp)
+    TLD = T.let(/\A([a-z]{2,}|xn--[a-z0-9-]+)\z/, Regexp)
+    IPV4 = T.let(/\A\d+(\.\d+){3}\z/, Regexp)
+
+    sig { params(input: String).returns(String) }
+    def self.normalize(input)
+      text = input.strip.downcase
+      invalid!(input, 'is empty') if text.empty?
+      invalid!(input, 'must be ASCII; use punycode ("xn--...") for internationalized names') unless text.ascii_only?
+
+      host = extract_host(input, text)
+      validate!(input, host)
+      host
+    end
+
+    sig { params(input: String, text: String).returns(String) }
+    def self.extract_host(input, text)
+      host = if text.include?('://') || text.start_with?('//')
+               URI.parse(text).host.to_s
+             else
+               text.split(%r{[/?#]}, 2).first.to_s
+             end
+      invalid!(input, 'is an IP address') if host.start_with?('[') || host.count(':') > 1
+      host.sub(/:\d*\z/, '').delete_suffix('.')
+    rescue URI::InvalidURIError
+      invalid!(input, 'is not a valid URL')
+    end
+
+    sig { params(input: String, host: String).void }
+    def self.validate!(input, host)
+      invalid!(input, 'has no hostname') if host.empty?
+      invalid!(input, 'is an IP address') if IPV4.match?(host)
+      invalid!(input, "is longer than #{MAX_LENGTH} characters") if host.length > MAX_LENGTH
+
+      validate_labels!(input, host.split('.', -1))
+    end
+
+    sig { params(input: String, labels: T::Array[String]).void }
+    def self.validate_labels!(input, labels)
+      invalid!(input, 'must contain a dot (e.g. "example.com")') if labels.length < 2
+      unless labels.all? { |label| LABEL.match?(label) }
+        invalid!(input, 'has an invalid label (1-63 letters, digits or inner hyphens)')
+      end
+      invalid!(input, 'has an invalid top-level domain') unless TLD.match?(labels.last.to_s)
+    end
+
+    sig { params(input: String, reason: String).returns(T.noreturn) }
+    def self.invalid!(input, reason)
+      raise ValidationError, "Invalid domain #{input.inspect}: #{reason}"
+    end
+
+    private_class_method :extract_host, :validate!, :validate_labels!, :invalid!
+  end
+end

data/lib/brand_logo/errors.rb

@@ -3,8 +3,6 @@
 
 module BrandLogo
   class Error < StandardError; end
-  class FetchError < Error; end
-  class ParseError < Error; end
   class ValidationError < Error; end
-  class NoIconFoundError < FetchError; end
+  class NoIconFoundError < Error; end
 end

data/lib/brand_logo/fetcher.rb

@@ -1,109 +1,106 @@
 # frozen_string_literal: true
 # typed: strict
 
-require 'sorbet-runtime'
-
 module BrandLogo
-  # Entry point for brand_logo retrieval.
-  #
-  # Composes a chain of strategies tried in order until one finds a valid icon.
-  # Dependencies (HTTP client, HTML parser, image analyzer) are instantiated once
-  # and shared across all strategies.
-  #
-  # Usage:
-  #   # Default configuration
-  #   icon = BrandLogo::Fetcher.new.fetch('github.com')
-  #
-  #   # Custom config
-  #   config = BrandLogo::Config.new(min_dimensions: { width: 32, height: 32 }, timeout: 5)
-  #   icon = BrandLogo::Fetcher.new(config: config).fetch('github.com')
-  #
-  #   # Custom strategy chain (OCP)
-  #   fetcher = BrandLogo::Fetcher.new(strategies: [MyCustomStrategy.new(config: config)])
-  #
-  #   # All icons from all strategies
-  #   icons = BrandLogo::Fetcher.new.fetch_all('github.com')
   class Fetcher
     extend T::Sig
 
-    DOMAIN_PATTERN = T.let(/\A[a-z0-9\-.]+\.[a-z]{2,}\z/i, Regexp)
+    CACHE_NAMESPACE = 'brand_logo/v2'
+
+    sig { returns(Config) }
+    attr_reader :config
 
     sig do
       params(
-        config: T.nilable(Config),
-        strategies: T.nilable(T::Array[Strategies::BaseStrategy])
+        config: Config,
+        http: T.nilable(Http::Client),
+        strategies: T.nilable(T::Array[Strategies::Base])
       ).void
     end
-    def initialize(config: nil, strategies: nil)
-      @config         = T.let(config || Config.new, Config)
-      @http_client    = T.let(RealHttpClient.new(@config), HttpClient)
-      @image_analyzer = T.let(FastimageAnalyzer.new, ImageAnalyzer)
-      @html_parser    = T.let(NokogiriParser.new, HtmlParser)
-      @strategies     = T.let(strategies || build_default_strategies, T::Array[Strategies::BaseStrategy])
+    def initialize(config: Config.new, http: nil, strategies: nil)
+      @config = config
+      @http = T.let(http || Http::RealClient.new(config), Http::Client)
+      @strategies = T.let(strategies || default_strategies, T::Array[Strategies::Base])
+      @ranker = T.let(Ranker.new(config), Ranker)
+      strategy_names = @strategies.map { |strategy| T.cast(strategy, Object).class.name }
+      @cache_digest = T.let(Digest::SHA256.hexdigest([config.cache_digest, strategy_names].inspect)[0, 12].to_s, String)
     end
 
-    # Returns the best icon found for the domain across all strategies.
-    # Raises NoIconFoundError if no strategy finds a valid icon.
-    sig { params(domain: String).returns(Icon) }
-    def fetch(domain)
-      validate_domain!(domain)
-      BrandLogo::Logging.logger.debug("Fetching brand_logo for: #{domain}")
-
-      @strategies.each do |strategy|
-        BrandLogo::Logging.logger.debug("Trying #{strategy.class.name}")
-        icon = strategy.fetch(domain)
-        return icon if icon
-      end
+    sig { params(input: String).returns(T.nilable(Icon)) }
+    def fetch(input)
+      fetch_domain(Domain.normalize(input))
+    end
 
-      raise NoIconFoundError, "No brand_logo found for #{domain}"
+    sig { params(input: String).returns(Icon) }
+    def fetch!(input)
+      fetch(input) || raise(NoIconFoundError, "No icon found for #{input.inspect}")
     end
 
-    # Returns all icons found across every strategy, deduplicated by URL.
-    sig { params(domain: String).returns(T::Array[Icon]) }
-    def fetch_all(domain)
-      validate_domain!(domain)
+    sig { params(input: String).returns(T::Array[Icon]) }
+    def fetch_all(input)
+      domain = Domain.normalize(input)
+      stored = cached("all/#{domain}") do
+        lookup = new_lookup(domain)
+        icons = lookup.all
+        [icons.map(&:to_h), ttl_for(found: !icons.empty?, lookup: lookup)]
+      end
+      Array(stored).map { |hash| Icon.from_h(hash) }
+    end
 
-      @strategies
-        .flat_map { |strategy| strategy.fetch_all(domain) }
-        .uniq(&:url)
+    sig { params(inputs: T::Array[String]).returns(T::Hash[String, T.nilable(Icon)]) }
+    def fetch_many(inputs)
+      domains = inputs.to_h { |input| [input, Domain.normalize(input)] }
+      unique = domains.values.uniq
+      icons = unique.zip(Concurrency.map(unique, size: @config.concurrency) { |domain| fetch_domain(domain) }).to_h
+      domains.transform_values { |domain| icons[domain] }
     end
 
     private
 
-    sig { params(domain: String).void }
-    def validate_domain!(domain)
-      return if domain.match?(DOMAIN_PATTERN)
+    sig { params(domain: String).returns(T.nilable(Icon)) }
+    def fetch_domain(domain)
+      stored = cached("best/#{domain}") do
+        lookup = new_lookup(domain)
+        icon = lookup.best
+        [icon ? icon.to_h : false, ttl_for(found: !icon.nil?, lookup: lookup)]
+      end
+      stored ? Icon.from_h(stored) : nil
+    end
+
+    sig { params(key: String, block: T.proc.returns([T.untyped, T.nilable(Integer)])).returns(T.untyped) }
+    def cached(key, &block)
+      cache = @config.cache
+      return yield.first unless cache
+
+      full_key = "#{CACHE_NAMESPACE}/#{key}/#{@cache_digest}"
+      stored = cache.read(full_key)
+      return stored unless stored.nil?
+
+      value, ttl = yield
+      cache.write(full_key, value, expires_in: ttl) if ttl&.positive?
+      value
+    end
+
+    # A lookup cut short by the deadline says little about the site, so it is never cached.
+    sig { params(found: T::Boolean, lookup: Lookup).returns(T.nilable(Integer)) }
+    def ttl_for(found:, lookup:)
+      return nil if lookup.timed_out?
+
+      found ? @config.cache_ttl : @config.negative_cache_ttl
+    end
 
-      raise ValidationError, "Invalid domain: #{domain.inspect}"
+    sig { params(domain: String).returns(Lookup) }
+    def new_lookup(domain)
+      context = Context.new(domain: domain, config: @config, http: @http)
+      Lookup.new(context: context, strategies: @strategies, ranker: @ranker)
     end
 
-    sig { returns(T::Array[Strategies::BaseStrategy]) }
-    def build_default_strategies
-      [
-        Strategies::ScrapingStrategy.new(
-          config: @config,
-          http_client: @http_client,
-          html_parser: @html_parser,
-          image_analyzer: @image_analyzer
-        ),
-        Strategies::MetaTagStrategy.new(
-          config: @config,
-          http_client: @http_client,
-          html_parser: @html_parser,
-          image_analyzer: @image_analyzer
-        ),
-        Strategies::ManifestStrategy.new(
-          config: @config,
-          http_client: @http_client,
-          html_parser: @html_parser,
-          image_analyzer: @image_analyzer
-        ),
-        Strategies::DuckduckgoStrategy.new(
-          config: @config,
-          http_client: @http_client,
-          image_analyzer: @image_analyzer
-        )
-      ]
+    sig { returns(T::Array[Strategies::Base]) }
+    def default_strategies
+      document = [Strategies::LinkTag.new, Strategies::JsonLd.new, Strategies::MetaTag.new]
+      remote = [Strategies::Manifest.new, Strategies::Browserconfig.new]
+      external = { google: Strategies::Google, duckduckgo: Strategies::Duckduckgo }
+      document + remote + @config.external_fallbacks.map { |name| external.fetch(name).new }
     end
   end
 end

data/lib/brand_logo/http/client.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+# typed: strict
+
+module BrandLogo
+  module Http
+    module Client
+      extend T::Sig
+      extend T::Helpers
+
+      interface!
+
+      # Never raises: any failure is nil, so strategies treat it as "no icon".
+      # `max_bytes` still caps the read because servers may ignore `range`.
+      sig do
+        abstract.params(
+          url: String,
+          deadline: Deadline,
+          max_bytes: Integer,
+          range: T.nilable(T::Range[Integer])
+        ).returns(T.nilable(Response))
+      end
+      def get(url, deadline:, max_bytes:, range: nil); end
+    end
+  end
+end

data/lib/brand_logo/http/real_client.rb

@@ -0,0 +1,110 @@
+# frozen_string_literal: true
+# typed: strict
+
+require 'http'
+require 'uri'
+
+module BrandLogo
+  module Http
+    class RealClient
+      extend T::Sig
+      include Client
+
+      ACCEPT = 'text/html,application/xhtml+xml,application/xml;q=0.9,image/*;q=0.8,*/*;q=0.5'
+      ACCEPT_LANGUAGE = 'en;q=0.8,*;q=0.5'
+      NETWORK_ERRORS = T.let([
+        HTTP::Error, OpenSSL::SSL::SSLError, SocketError, IOError, SystemCallError, Timeout::Error, URI::Error
+      ].freeze, T::Array[T.class_of(Exception)])
+      REDIRECT_STATUSES = T.let([301, 302, 303, 307, 308].freeze, T::Array[Integer])
+
+      sig { params(config: Config).void }
+      def initialize(config)
+        @config = config
+      end
+
+      sig do
+        override.params(
+          url: String,
+          deadline: Deadline,
+          max_bytes: Integer,
+          range: T.nilable(T::Range[Integer])
+        ).returns(T.nilable(Response))
+      end
+      def get(url, deadline:, max_bytes:, range: nil)
+        current = url
+        (@config.max_hops + 1).times do
+          return nil if deadline.expired? || !allowed?(current)
+
+          result = request(current, deadline, max_bytes, range)
+          return result if result.nil? || result.is_a?(Response)
+
+          current = result
+        end
+        log("too many redirects for #{url}")
+        nil
+      rescue *NETWORK_ERRORS => e
+        log("#{url}: #{e.class}: #{e.message}")
+        nil
+      end
+
+      private
+
+      sig { params(url: String).returns(T::Boolean) }
+      def allowed?(url)
+        return true if UrlGuard.allowed?(url, block_private_ips: @config.block_private_ips)
+
+        log("blocked #{url}")
+        false
+      end
+
+      sig do
+        params(url: String, deadline: Deadline, max_bytes: Integer, range: T.nilable(T::Range[Integer]))
+          .returns(T.nilable(T.any(Response, String)))
+      end
+      def request(url, deadline, max_bytes, range)
+        client = HTTP.timeout([@config.timeout, deadline.remaining].min).headers(headers(range))
+        response = client.get(url)
+        status = response.status.to_i
+        return redirect_target(url, response) if REDIRECT_STATUSES.include?(status)
+
+        unless response.status.success?
+          log("#{url}: HTTP #{status}")
+          return nil
+        end
+
+        Response.new(url: url, body: read_body(response, max_bytes))
+      ensure
+        client&.close
+      end
+
+      sig { params(url: String, response: HTTP::Response).returns(T.nilable(String)) }
+      def redirect_target(url, response)
+        location = response.headers['Location']&.to_s
+        return nil if location.nil? || location.empty?
+
+        URI.join(url, location).to_s
+      end
+
+      sig { params(response: HTTP::Response, max_bytes: Integer).returns(String) }
+      def read_body(response, max_bytes)
+        buffer = String.new(encoding: Encoding::BINARY)
+        response.body.each do |chunk|
+          buffer << chunk.b
+          break if buffer.bytesize >= max_bytes
+        end
+        buffer.bytesize > max_bytes ? T.must(buffer.byteslice(0, max_bytes)) : buffer
+      end
+
+      sig { params(range: T.nilable(T::Range[Integer])).returns(T::Hash[String, String]) }
+      def headers(range)
+        base = { 'User-Agent' => @config.user_agent, 'Accept' => ACCEPT, 'Accept-Language' => ACCEPT_LANGUAGE }
+        range ? base.merge('Range' => "bytes=#{range.min}-#{range.max}") : base
+      end
+
+      sig { params(message: String).void }
+      def log(message)
+        Logging.logger.debug("[BrandLogo] #{message}")
+      end
+    end
+  end
+end

data/lib/brand_logo/http/response.rb

@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+# typed: strict
+
+module BrandLogo
+  module Http
+    Response = Data.define(:url, :body)
+  end
+end

data/lib/brand_logo/http/url_guard.rb

@@ -0,0 +1,68 @@
+# frozen_string_literal: true
+# typed: strict
+
+require 'ipaddr'
+require 'resolv'
+require 'socket'
+require 'uri'
+
+module BrandLogo
+  module Http
+    module UrlGuard
+      extend T::Sig
+
+      SCHEMES = T.let(%w[http https].freeze, T::Array[String])
+      BLOCKED_RANGES = T.let(
+        %w[
+          0.0.0.0/8 100.64.0.0/10 192.0.0.0/24 192.0.2.0/24 198.18.0.0/15 198.51.100.0/24
+          203.0.113.0/24 224.0.0.0/4 240.0.0.0/4 ::/128 64:ff9b::/96 100::/64 fc00::/7 fe80::/10 ff00::/8
+        ].map { |cidr| IPAddr.new(cidr) }.freeze,
+        T::Array[IPAddr]
+      )
+
+      sig { params(url: String, block_private_ips: T::Boolean).returns(T::Boolean) }
+      def self.allowed?(url, block_private_ips:)
+        host = web_host(URI.parse(url))
+        return false unless host
+        return true unless block_private_ips
+
+        addresses = resolve(host)
+        !addresses.empty? && addresses.none? { |address| blocked?(address) }
+      rescue URI::Error
+        false
+      end
+
+      sig { params(uri: URI::Generic).returns(T.nilable(String)) }
+      def self.web_host(uri)
+        host = uri.hostname
+        host if SCHEMES.include?(uri.scheme&.downcase) && host && !host.empty?
+      end
+
+      sig { params(host: String).returns(T::Array[IPAddr]) }
+      def self.resolve(host)
+        literal = parse_ip(host)
+        return [literal] if literal
+
+        Addrinfo.getaddrinfo(host, nil, nil, :STREAM).filter_map { |info| parse_ip(info.ip_address) }
+      rescue SocketError
+        []
+      end
+
+      sig { params(value: String).returns(T.nilable(IPAddr)) }
+      def self.parse_ip(value)
+        IPAddr.new(value.sub(/%.*\z/, ''))
+      rescue IPAddr::Error
+        nil
+      end
+
+      sig { params(address: IPAddr).returns(T::Boolean) }
+      def self.blocked?(address)
+        address = address.native if address.ipv4_mapped?
+        address.private? || address.loopback? || address.link_local? ||
+          BLOCKED_RANGES.any? { |range| range.family == address.family && range.include?(address) }
+      end
+
+      private_class_method :resolve, :parse_ip, :blocked?
+    end
+  end
+end

data/lib/brand_logo/icon.rb

@@ -1,34 +1,55 @@
 # frozen_string_literal: true
 # typed: strict
 
-require 'sorbet-runtime'
-
 module BrandLogo
-  # Represents a brand_logo icon with its URL, dimensions and format
-  # Used to store and return brand_logo information across the gem
-  class Icon
+  Icon = Data.define(:url, :format, :dimensions, :source, :kind, :media) do
     extend T::Sig
 
-    sig { returns(String) }
-    attr_reader :url
-
-    sig { returns(T::Hash[Symbol, T.nilable(Integer)]) }
-    attr_reader :dimensions
-
-    sig { returns(String) }
-    attr_reader :format
-
     sig do
       params(
         url: String,
-        dimensions: T::Hash[Symbol, T.nilable(Integer)],
-        format: String
+        source: Symbol,
+        format: T.nilable(Symbol),
+        dimensions: Dimensions,
+        kind: Symbol,
+        media: T.nilable(String)
       ).void
     end
-    def initialize(url:, dimensions:, format:)
-      @url = url
-      @dimensions = dimensions
-      @format = format
+    def initialize(url:, source:, format: nil, dimensions: Dimensions.new, kind: :icon, media: nil)
+      super
+    end
+
+    sig { params(hash: T::Hash[T.any(String, Symbol), T.untyped]).returns(Icon) }
+    def self.from_h(hash)
+      data = hash.transform_keys(&:to_sym)
+      new(
+        url: data.fetch(:url),
+        source: data.fetch(:source).to_sym,
+        format: data[:format]&.to_sym,
+        dimensions: Dimensions.new(width: data[:width], height: data[:height]),
+        kind: data.fetch(:kind, :icon).to_sym,
+        media: data[:media]
+      )
+    end
+
+    sig { returns(T.nilable(Integer)) }
+    def width
+      dimensions.width
+    end
+
+    sig { returns(T.nilable(Integer)) }
+    def height
+      dimensions.height
+    end
+
+    sig { returns(T::Boolean) }
+    def svg?
+      format == :svg
+    end
+
+    sig { returns(T::Hash[Symbol, T.untyped]) }
+    def to_h
+      { url: url, format: format, width: width, height: height, source: source, kind: kind, media: media }
     end
   end
 end

data/lib/brand_logo/image_format.rb

@@ -0,0 +1,53 @@
+# frozen_string_literal: true
+# typed: strict
+
+require 'uri'
+
+module BrandLogo
+  module ImageFormat
+    extend T::Sig
+
+    MIME_TYPES = T.let({
+      'image/x-icon' => :ico,
+      'image/vnd.microsoft.icon' => :ico,
+      'image/ico' => :ico,
+      'image/png' => :png,
+      'image/svg+xml' => :svg,
+      'image/jpeg' => :jpg,
+      'image/jpg' => :jpg,
+      'image/webp' => :webp,
+      'image/gif' => :gif,
+      'image/avif' => :avif,
+      'image/bmp' => :bmp
+    }.freeze, T::Hash[String, Symbol])
+
+    EXTENSIONS = T.let({
+      'ico' => :ico, 'cur' => :ico, 'png' => :png, 'svg' => :svg, 'jpg' => :jpg, 'jpeg' => :jpg,
+      'webp' => :webp, 'gif' => :gif, 'avif' => :avif, 'bmp' => :bmp
+    }.freeze, T::Hash[String, Symbol])
+
+    FASTIMAGE_TYPES = T.let({ jpeg: :jpg, cur: :ico }.freeze, T::Hash[Symbol, Symbol])
+
+    sig { params(mime: T.untyped).returns(T.nilable(Symbol)) }
+    def self.from_mime(mime)
+      MIME_TYPES[mime.to_s.split(';').first.to_s.strip.downcase]
+    end
+
+    sig { params(url: String).returns(T.nilable(Symbol)) }
+    def self.from_url(url)
+      return from_mime(url[/\Adata:([^;,]+)/i, 1]) if url.start_with?('data:')
+
+      path = URI.parse(url).path.to_s
+      EXTENSIONS[File.extname(path).delete_prefix('.').downcase]
+    rescue URI::Error
+      nil
+    end
+
+    sig { params(type: T.nilable(Symbol)).returns(T.nilable(Symbol)) }
+    def self.from_fastimage(type)
+      return nil if type.nil?
+
+      FASTIMAGE_TYPES.fetch(type, type)
+    end
+  end
+end