ficon 0.3 → 0.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 6b300144548c846bb2d309ca04ca78198f9e37682e50f08ac2f60e95f54e84f5
-  data.tar.gz: 90e5b7804b8e6c44e8c361c585e03a2553724322cebf84846c66c73c1652a234
+  metadata.gz: a9f5f2c6b36b5360a88f4eac14c07c96f7fefb9674fb6bed0bdf4a7be63ad2d0
+  data.tar.gz: f85066461f2384ed8e2ed3d302e5e3ffe537178dfaf40c128e056dd16c1e3027
 SHA512:
-  metadata.gz: 6781246417b9d5cc5e1791acae5232dcd13ea9a934f0f977929fd57a5fa348dff96143b2acf327bd3ea7626e0eea65c92b585f4bd544c3fcd7764de76eee121b
-  data.tar.gz: 0a2b83a50bdbc340e8877028f77f99de6ca23e9bdfa15b237f0d47390394ba3f9165ebfdf2a625bffdbebca40789cac8ce2cc3b6d9c6cf11c79ae2f23f8aca19
+  metadata.gz: 7ac7f80ea66b97249dccbfcdd4c7adf82b741125c23dbd92339346fb63e772bcfcea893279cca1b05b100946d567ee4c82bdfc9ba85dcd4fd10c881bef987ae9
+  data.tar.gz: 8c4421c19189b779f19aca1699521e600c25f4a17807dc22a546913a28e0d230263642d8610ef68f81d548f66be4cff1654dc9a12a644319c281432555428b4e

data/lib/ficon/cache.rb

@@ -40,6 +40,33 @@ class Ficon
       db.execute("UPDATE urls SET not_before=? WHERE url=?", [_value, @url])
     end
 
+    def status
+      db.execute("select status from urls where url=? limit 1", @url).first&.first
+    end
+
+    def status=(_value)
+      db.execute("INSERT OR IGNORE INTO urls (url, status) VALUES (?, ?)", [@url, _value])
+      db.execute("UPDATE urls SET status=? WHERE url=?", [_value, @url])
+    end
+
+    def retry_count
+      db.execute("select retry_count from urls where url=? limit 1", @url).first&.first || 0
+    end
+
+    def retry_count=(_value)
+      db.execute("INSERT OR IGNORE INTO urls (url, retry_count) VALUES (?, ?)", [@url, _value])
+      db.execute("UPDATE urls SET retry_count=? WHERE url=?", [_value, @url])
+    end
+
+    def last_attempt
+      db.execute("select last_attempt from urls where url=? limit 1", @url).first&.first
+    end
+
+    def last_attempt=(_value)
+      db.execute("INSERT OR IGNORE INTO urls (url, last_attempt) VALUES (?, ?)", [@url, _value])
+      db.execute("UPDATE urls SET last_attempt=? WHERE url=?", [_value, @url])
+    end
+
     def self.db_file
       if ENV["FICON_DB"].nil?
         File.expand_path("~/.ficon.db")
@@ -49,8 +76,12 @@ class Ficon
     end
 
     def self.setup_cache(db)
-      db.execute("CREATE TABLE urls(url, etag, not_before, data)")
+      db.execute("CREATE TABLE urls(url, etag, not_before, data, status, retry_count, last_attempt)")
       db.execute("CREATE UNIQUE INDEX `url` ON `urls` (`url`)")
     end
+
+    def self.clear_cache
+      File.delete(db_file) if File.exist?(db_file)
+    end
   end
 end

data/lib/ficon/version.rb

@@ -1,3 +1,3 @@
 class Ficon
-  VERSION = "0.3"
+  VERSION = "0.5"
 end

data/lib/ficon.rb

@@ -2,6 +2,7 @@ require "net/http"
 require "nokogiri"
 require "uri"
 require "addressable/uri"
+require "resolv"
 require "debug"
 
 require_relative "ficon/version"
@@ -9,14 +10,21 @@ require_relative "ficon/image"
 require_relative "ficon/cache"
 
 class Ficon
-  attr_reader :site, :final_uri
+  attr_reader :site, :final_uri, :url_status
   attr_accessor :user_agent
+  
+  # URL health status constants
+  ALIVE = 'alive'
+  DEAD = 'dead'
+  SICK = 'sick'
+  BLOCKED = 'blocked'
 
   def initialize(uri, user_agent: nil)
     @uri = Addressable::URI.heuristic_parse(uri)
     @final_uri = @uri
     @site = {}
-    @user_agent = user_agent || "Ficon/#{VERSION} (Ruby icon finder; https://github.com/dkam/ficon)"
+    @url_status = nil
+    @user_agent = user_agent || "FiconBot/#{VERSION} (Ruby icon finder; https://github.com/dkam/ficon)"
     process
   end
 
@@ -56,9 +64,15 @@ class Ficon
   end
 
   def process
-    @site[:images] = self.class.site_images(@uri, doc) || []
-    @site[:page_images] = self.class.page_images(@uri, doc) || []
-    other_page_data
+    document = doc
+    if document
+      @site[:images] = self.class.site_images(@uri, document) || []
+      @site[:page_images] = self.class.page_images(@uri, document) || []
+      other_page_data(document)
+    else
+      @site[:images] = []
+      @site[:page_images] = []
+    end
     nil
   end
 
@@ -70,26 +84,27 @@ class Ficon
     report_lines << "Page description: #{@site[:description]}"
     report_lines << "Final URL: #{@final_uri}" if @final_uri.to_s != @uri.to_s
     report_lines << "Canonical URL: #{@site[:canonical]}" if @site[:canonical]
+    report_lines << "URL Status: #{@url_status}" if @url_status
     report_lines.join("\n") + "\n"
   end
 
-  def site_icons = @site[:images]
-
-  def site_icon = site_icons&.first
+  def site_icons = @site[:images] || []
 
-  def page_images = @site[:page_images]
-
-  def page_image = page_images&.first
+  def page_images = @site[:page_images] || []
 
   def title = @site[:title]
 
   def description = @site[:description]
 
-  def other_page_data
-    @site[:title] = doc.at_xpath("//meta[@property='og:title']/@content")&.value || @doc.at_xpath("//title")&.text&.strip
-    @site[:description] = doc.at_xpath("//meta[@property='og:description']/@content")&.value
-    canonical = doc.at_xpath("//link[@rel='canonical']/@href")&.value
-    @site[:canonical] = canonical unless canonical == @url
+  def self.clear_cache
+    Cache.clear_cache
+  end
+
+  def other_page_data(document)
+    @site[:title] = document.at_xpath("//meta[@property='og:title']/@content")&.value || document.at_xpath("//title")&.text&.strip
+    @site[:description] = document.at_xpath("//meta[@property='og:description']/@content")&.value
+    canonical = document.at_xpath("//link[@rel='canonical']/@href")&.value
+    @site[:canonical] = canonical unless canonical == @uri.to_s
   end
 
   def self.site_images(uri, doc)
@@ -124,12 +139,43 @@ class Ficon
     parsed_candidate.to_s
   end
 
+  def classify_response_status(response)
+    case response.code.to_i
+    when 200..299
+      ALIVE
+    when 404, 410
+      DEAD
+    when 401, 403, 429
+      BLOCKED
+    when 500..599
+      SICK
+    else
+      SICK
+    end
+  end
+
+  def classify_exception_status(exception)
+    case exception
+    when SocketError, Resolv::ResolvError
+      DEAD  # DNS resolution failures
+    when Net::HTTPError, Timeout::Error, Errno::ECONNREFUSED
+      SICK  # Network issues worth retrying
+    when OpenSSL::SSL::SSLError
+      SICK  # SSL certificate errors
+    else
+      SICK  # Default to retryable for unknown errors
+    end
+  end
+
   private
 
   def fetch_url(uri, redirect_limit = 5)
     uri = URI(uri) unless uri.is_a?(URI)
-
-    raise "Too many redirects" if redirect_limit <= 0
+    
+    if redirect_limit <= 0
+      @url_status = DEAD
+      raise "Too many redirects"
+    end
 
     Net::HTTP.start(uri.host, uri.port, use_ssl: uri.scheme == "https") do |http|
       http.read_timeout = 10
@@ -137,7 +183,10 @@ class Ficon
       request = Net::HTTP::Get.new(uri)
       request["User-Agent"] = @user_agent
       response = http.request(request)
-
+      
+      # Set status based on response
+      @url_status = classify_response_status(response)
+      
       case response
       when Net::HTTPRedirection
         location = response["location"]
@@ -149,10 +198,11 @@ class Ficon
       else
         @final_uri = Addressable::URI.parse(uri.to_s)
       end
-
+      
       response
     end
-  rescue Net::HTTPError, SocketError, Timeout::Error => e
+  rescue => e
+    @url_status = classify_exception_status(e)
     puts "Failed to fetch #{uri}: #{e.inspect}"
     nil
   end

data/test/ficon_test.rb

@@ -1,5 +1,6 @@
 #require 'rubygems'
 require 'debug'
+require 'resolv'
 
 require "minitest/autorun"
 
@@ -58,7 +59,7 @@ class FiconTest < Minitest::Test
   def test_custom_user_agent
     # Test default user agent
     ficon_default = Ficon.new('https://example.com')
-    assert_match(/^Ficon\/0\.2/, ficon_default.user_agent)
+    assert_match(/^FiconBot\/0\.\d+/, ficon_default.user_agent)
     
     # Test custom user agent
     custom_agent = 'MyApp/1.0 (Custom Bot)'
@@ -69,4 +70,53 @@ class FiconTest < Minitest::Test
     ficon_custom.user_agent = 'Changed/2.0'
     assert_equal 'Changed/2.0', ficon_custom.user_agent
   end
+
+  def test_response_status_classification
+    ficon = Ficon.new('https://example.com')
+    
+    # Test ALIVE status (2xx)
+    assert_equal Ficon::ALIVE, ficon.classify_response_status(mock_response(200))
+    assert_equal Ficon::ALIVE, ficon.classify_response_status(mock_response(201))
+    assert_equal Ficon::ALIVE, ficon.classify_response_status(mock_response(299))
+    
+    # Test DEAD status (404, 410)
+    assert_equal Ficon::DEAD, ficon.classify_response_status(mock_response(404))
+    assert_equal Ficon::DEAD, ficon.classify_response_status(mock_response(410))
+    
+    # Test BLOCKED status (401, 403, 429)
+    assert_equal Ficon::BLOCKED, ficon.classify_response_status(mock_response(401))
+    assert_equal Ficon::BLOCKED, ficon.classify_response_status(mock_response(403))
+    assert_equal Ficon::BLOCKED, ficon.classify_response_status(mock_response(429))
+    
+    # Test SICK status (5xx and others)
+    assert_equal Ficon::SICK, ficon.classify_response_status(mock_response(500))
+    assert_equal Ficon::SICK, ficon.classify_response_status(mock_response(502))
+    assert_equal Ficon::SICK, ficon.classify_response_status(mock_response(503))
+    assert_equal Ficon::SICK, ficon.classify_response_status(mock_response(300)) # Other codes default to SICK
+  end
+
+  def test_exception_status_classification
+    ficon = Ficon.new('https://example.com')
+    
+    # Test DEAD status (DNS and resolution errors)
+    assert_equal Ficon::DEAD, ficon.classify_exception_status(SocketError.new)
+    assert_equal Ficon::DEAD, ficon.classify_exception_status(Resolv::ResolvError.new)
+    
+    # Test SICK status (network and timeout errors)
+    assert_equal Ficon::SICK, ficon.classify_exception_status(Timeout::Error.new)
+    assert_equal Ficon::SICK, ficon.classify_exception_status(Errno::ECONNREFUSED.new)
+    assert_equal Ficon::SICK, ficon.classify_exception_status(OpenSSL::SSL::SSLError.new)
+    assert_equal Ficon::SICK, ficon.classify_exception_status(Net::HTTPError.new('error', nil))
+    
+    # Test default to SICK for unknown exceptions
+    assert_equal Ficon::SICK, ficon.classify_exception_status(StandardError.new)
+  end
+
+  private
+
+  def mock_response(code)
+    response = Object.new
+    response.define_singleton_method(:code) { code }
+    response
+  end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: ficon
 version: !ruby/object:Gem::Version
-  version: '0.3'
+  version: '0.5'
 platform: ruby
 authors:
 - Dan Milne