ficon 0.2 → 0.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 97c762adf3cdb126d047b66c426c3f388a3a8c61e2c6c009cff2f0dca3c7f219
-  data.tar.gz: 53a177388f199f7133991f1067ad1a71d747c5d84c99b88d379d7878a3196eb5
+  metadata.gz: 4e8577eb04ba2dceefb974b436520ff73b8c37f6beb04c85ac62f02e774bbc94
+  data.tar.gz: 4196a3eb41905e40f0285eb132b3ae4d9e9c2c00f002584f6dc0b667893842db
 SHA512:
-  metadata.gz: 2af1238cf43c03e84081ddfeeb69a545613e0348b33dab4aa3670d8e152e9ce4089f9b8c892764ecb143cf2aaa90b6afa4fd7b844bb611e98a8b18b27ce161b6
-  data.tar.gz: 6eac859ed1529c3f22fcdaf5634a8e66ae1c625da7fbc0a1bce80afe53fc2a2a15145ac827cf65de414e68b7ea86be6cf25b808102606ab44f77def6a20124c4
+  metadata.gz: afbea646a672f8654bd8ba76a628c373ca1c40dbf051af8937656f755f65872031a6fc88e6a223914469a5baf10e303fc47e770ddb3abbf70305c1e08ebe1714
+  data.tar.gz: b08f3e0d30b5f93f503d03b5222938ab37e7c5fb423b5cbcf2df3fe217ed791aec4dc8cb8a32e540e41cd9bf06d7ea62b3bc062b159aa64452caa992c069de29

data/lib/ficon/cache.rb

@@ -40,6 +40,33 @@ class Ficon
       db.execute("UPDATE urls SET not_before=? WHERE url=?", [_value, @url])
     end
 
+    def status
+      db.execute("select status from urls where url=? limit 1", @url).first&.first
+    end
+
+    def status=(_value)
+      db.execute("INSERT OR IGNORE INTO urls (url, status) VALUES (?, ?)", [@url, _value])
+      db.execute("UPDATE urls SET status=? WHERE url=?", [_value, @url])
+    end
+
+    def retry_count
+      db.execute("select retry_count from urls where url=? limit 1", @url).first&.first || 0
+    end
+
+    def retry_count=(_value)
+      db.execute("INSERT OR IGNORE INTO urls (url, retry_count) VALUES (?, ?)", [@url, _value])
+      db.execute("UPDATE urls SET retry_count=? WHERE url=?", [_value, @url])
+    end
+
+    def last_attempt
+      db.execute("select last_attempt from urls where url=? limit 1", @url).first&.first
+    end
+
+    def last_attempt=(_value)
+      db.execute("INSERT OR IGNORE INTO urls (url, last_attempt) VALUES (?, ?)", [@url, _value])
+      db.execute("UPDATE urls SET last_attempt=? WHERE url=?", [_value, @url])
+    end
+
     def self.db_file
       if ENV["FICON_DB"].nil?
         File.expand_path("~/.ficon.db")
@@ -49,8 +76,12 @@ class Ficon
     end
 
     def self.setup_cache(db)
-      db.execute("CREATE TABLE urls(url, etag, not_before, data)")
+      db.execute("CREATE TABLE urls(url, etag, not_before, data, status, retry_count, last_attempt)")
       db.execute("CREATE UNIQUE INDEX `url` ON `urls` (`url`)")
     end
+
+    def self.clear_cache
+      File.delete(db_file) if File.exist?(db_file)
+    end
   end
 end

data/lib/ficon/version.rb

@@ -1,3 +1,3 @@
 class Ficon
-  VERSION = "0.2"
+  VERSION = "0.4"
 end

data/lib/ficon.rb

@@ -2,6 +2,7 @@ require "net/http"
 require "nokogiri"
 require "uri"
 require "addressable/uri"
+require "resolv"
 require "debug"
 
 require_relative "ficon/version"
@@ -9,25 +10,35 @@ require_relative "ficon/image"
 require_relative "ficon/cache"
 
 class Ficon
-  attr_reader :site
+  attr_reader :site, :final_uri, :url_status
   attr_accessor :user_agent
+  
+  # URL health status constants
+  ALIVE = 'alive'
+  DEAD = 'dead'
+  SICK = 'sick'
+  BLOCKED = 'blocked'
 
   def initialize(uri, user_agent: nil)
     @uri = Addressable::URI.heuristic_parse(uri)
+    @final_uri = @uri
     @site = {}
-    @user_agent = user_agent || "Ficon/#{VERSION} (Ruby icon finder; https://github.com/dkam/ficon)"
+    @url_status = nil
+    @user_agent = user_agent || "FiconBot/#{VERSION} (Ruby icon finder; https://github.com/dkam/ficon)"
     process
   end
 
   def doc
-    cache = Cache.new(@uri)
+    # First try to fetch to determine final URL
+    response = fetch_url(@uri) unless @data
+    return nil if response.nil? && @data.nil?
 
-    @data ||= cache.data
+    # Use final URL for caching
+    cache = Cache.new(@final_uri)
 
-    if @data.nil?
-      response = fetch_url(@uri)
-      return nil unless response
+    @data ||= cache.data
 
+    if @data.nil? && response
       @data = response.body.force_encoding("UTF-8")
       cache.data = @data
       cache.etag = response["etag"] if response["etag"]
@@ -60,29 +71,31 @@ class Ficon
   end
 
   def report
-    <<~REPORT
-      Site icon: #{@site[:images].first}
-      Page icon: #{@site[:page_images].first}
-      Page title: #{@site[:title]}
-      Page description: #{@site[:description]}
-      Canonical URL: #{@site[:canonical]}
-    REPORT
+    report_lines = []
+    report_lines << "Site icon: #{@site[:images].first}"
+    report_lines << "Page icon: #{@site[:page_images].first}"
+    report_lines << "Page title: #{@site[:title]}"
+    report_lines << "Page description: #{@site[:description]}"
+    report_lines << "Final URL: #{@final_uri}" if @final_uri.to_s != @uri.to_s
+    report_lines << "Canonical URL: #{@site[:canonical]}" if @site[:canonical]
+    report_lines << "URL Status: #{@url_status}" if @url_status
+    report_lines.join("\n") + "\n"
   end
 
-  def site_icons
-    @site[:images]
-  end
+  def site_icons = @site[:images]
 
-  def page_images
-    @site[:page_images]
-  end
+  def site_icon = site_icons&.first
 
-  def title
-    @site[:title]
-  end
+  def page_images = @site[:page_images]
 
-  def description
-    @site[:description]
+  def page_image = page_images&.first
+
+  def title = @site[:title]
+
+  def description = @site[:description]
+
+  def self.clear_cache
+    Cache.clear_cache
   end
 
   def other_page_data
@@ -126,18 +139,69 @@ class Ficon
 
   private
 
-  def fetch_url(uri)
+  def fetch_url(uri, redirect_limit = 5)
     uri = URI(uri) unless uri.is_a?(URI)
+    
+    if redirect_limit <= 0
+      @url_status = DEAD
+      raise "Too many redirects"
+    end
 
     Net::HTTP.start(uri.host, uri.port, use_ssl: uri.scheme == "https") do |http|
       http.read_timeout = 10
       http.open_timeout = 5
       request = Net::HTTP::Get.new(uri)
       request["User-Agent"] = @user_agent
-      http.request(request)
+      response = http.request(request)
+      
+      # Set status based on response
+      @url_status = classify_response_status(response)
+      
+      case response
+      when Net::HTTPRedirection
+        location = response["location"]
+        if location
+          new_uri = URI.join(uri.to_s, location)
+          @final_uri = Addressable::URI.parse(new_uri.to_s)
+          return fetch_url(new_uri, redirect_limit - 1)
+        end
+      else
+        @final_uri = Addressable::URI.parse(uri.to_s)
+      end
+      
+      response
     end
-  rescue Net::HTTPError, SocketError, Timeout::Error => e
+  rescue => e
+    @url_status = classify_exception_status(e)
     puts "Failed to fetch #{uri}: #{e.inspect}"
     nil
   end
+
+  def classify_response_status(response)
+    case response.code.to_i
+    when 200..299
+      ALIVE
+    when 404, 410
+      DEAD
+    when 401, 403, 429
+      BLOCKED
+    when 500..599
+      SICK
+    else
+      SICK
+    end
+  end
+
+  def classify_exception_status(exception)
+    case exception
+    when SocketError, Resolv::ResolutionError
+      DEAD  # DNS resolution failures
+    when Net::HTTPError, Timeout::Error, Errno::ECONNREFUSED
+      SICK  # Network issues worth retrying
+    when OpenSSL::SSL::SSLError
+      SICK  # SSL certificate errors
+    else
+      SICK  # Default to retryable for unknown errors
+    end
+  end
 end

data/test/ficon_test.rb

@@ -58,7 +58,7 @@ class FiconTest < Minitest::Test
   def test_custom_user_agent
     # Test default user agent
     ficon_default = Ficon.new('https://example.com')
-    assert_match(/^Ficon\/0\.2/, ficon_default.user_agent)
+    assert_match(/^FiconBot\/0\.\d+/, ficon_default.user_agent)
     
     # Test custom user agent
     custom_agent = 'MyApp/1.0 (Custom Bot)'

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: ficon
 version: !ruby/object:Gem::Version
-  version: '0.2'
+  version: '0.4'
 platform: ruby
 authors:
 - Dan Milne