ficon 0.3 → 0.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 6b300144548c846bb2d309ca04ca78198f9e37682e50f08ac2f60e95f54e84f5
-  data.tar.gz: 90e5b7804b8e6c44e8c361c585e03a2553724322cebf84846c66c73c1652a234
+  metadata.gz: 4e8577eb04ba2dceefb974b436520ff73b8c37f6beb04c85ac62f02e774bbc94
+  data.tar.gz: 4196a3eb41905e40f0285eb132b3ae4d9e9c2c00f002584f6dc0b667893842db
 SHA512:
-  metadata.gz: 6781246417b9d5cc5e1791acae5232dcd13ea9a934f0f977929fd57a5fa348dff96143b2acf327bd3ea7626e0eea65c92b585f4bd544c3fcd7764de76eee121b
-  data.tar.gz: 0a2b83a50bdbc340e8877028f77f99de6ca23e9bdfa15b237f0d47390394ba3f9165ebfdf2a625bffdbebca40789cac8ce2cc3b6d9c6cf11c79ae2f23f8aca19
+  metadata.gz: afbea646a672f8654bd8ba76a628c373ca1c40dbf051af8937656f755f65872031a6fc88e6a223914469a5baf10e303fc47e770ddb3abbf70305c1e08ebe1714
+  data.tar.gz: b08f3e0d30b5f93f503d03b5222938ab37e7c5fb423b5cbcf2df3fe217ed791aec4dc8cb8a32e540e41cd9bf06d7ea62b3bc062b159aa64452caa992c069de29

data/lib/ficon/cache.rb

@@ -40,6 +40,33 @@ class Ficon
       db.execute("UPDATE urls SET not_before=? WHERE url=?", [_value, @url])
     end
 
+    def status
+      db.execute("select status from urls where url=? limit 1", @url).first&.first
+    end
+
+    def status=(_value)
+      db.execute("INSERT OR IGNORE INTO urls (url, status) VALUES (?, ?)", [@url, _value])
+      db.execute("UPDATE urls SET status=? WHERE url=?", [_value, @url])
+    end
+
+    def retry_count
+      db.execute("select retry_count from urls where url=? limit 1", @url).first&.first || 0
+    end
+
+    def retry_count=(_value)
+      db.execute("INSERT OR IGNORE INTO urls (url, retry_count) VALUES (?, ?)", [@url, _value])
+      db.execute("UPDATE urls SET retry_count=? WHERE url=?", [_value, @url])
+    end
+
+    def last_attempt
+      db.execute("select last_attempt from urls where url=? limit 1", @url).first&.first
+    end
+
+    def last_attempt=(_value)
+      db.execute("INSERT OR IGNORE INTO urls (url, last_attempt) VALUES (?, ?)", [@url, _value])
+      db.execute("UPDATE urls SET last_attempt=? WHERE url=?", [_value, @url])
+    end
+
     def self.db_file
       if ENV["FICON_DB"].nil?
         File.expand_path("~/.ficon.db")
@@ -49,8 +76,12 @@ class Ficon
     end
 
     def self.setup_cache(db)
-      db.execute("CREATE TABLE urls(url, etag, not_before, data)")
+      db.execute("CREATE TABLE urls(url, etag, not_before, data, status, retry_count, last_attempt)")
       db.execute("CREATE UNIQUE INDEX `url` ON `urls` (`url`)")
     end
+
+    def self.clear_cache
+      File.delete(db_file) if File.exist?(db_file)
+    end
   end
 end

data/lib/ficon/version.rb

@@ -1,3 +1,3 @@
 class Ficon
-  VERSION = "0.3"
+  VERSION = "0.4"
 end

data/lib/ficon.rb

@@ -2,6 +2,7 @@ require "net/http"
 require "nokogiri"
 require "uri"
 require "addressable/uri"
+require "resolv"
 require "debug"
 
 require_relative "ficon/version"
@@ -9,14 +10,21 @@ require_relative "ficon/image"
 require_relative "ficon/cache"
 
 class Ficon
-  attr_reader :site, :final_uri
+  attr_reader :site, :final_uri, :url_status
   attr_accessor :user_agent
+  
+  # URL health status constants
+  ALIVE = 'alive'
+  DEAD = 'dead'
+  SICK = 'sick'
+  BLOCKED = 'blocked'
 
   def initialize(uri, user_agent: nil)
     @uri = Addressable::URI.heuristic_parse(uri)
     @final_uri = @uri
     @site = {}
-    @user_agent = user_agent || "Ficon/#{VERSION} (Ruby icon finder; https://github.com/dkam/ficon)"
+    @url_status = nil
+    @user_agent = user_agent || "FiconBot/#{VERSION} (Ruby icon finder; https://github.com/dkam/ficon)"
     process
   end
 
@@ -70,6 +78,7 @@ class Ficon
     report_lines << "Page description: #{@site[:description]}"
     report_lines << "Final URL: #{@final_uri}" if @final_uri.to_s != @uri.to_s
     report_lines << "Canonical URL: #{@site[:canonical]}" if @site[:canonical]
+    report_lines << "URL Status: #{@url_status}" if @url_status
     report_lines.join("\n") + "\n"
   end
 
@@ -85,6 +94,10 @@ class Ficon
 
   def description = @site[:description]
 
+  def self.clear_cache
+    Cache.clear_cache
+  end
+
   def other_page_data
     @site[:title] = doc.at_xpath("//meta[@property='og:title']/@content")&.value || @doc.at_xpath("//title")&.text&.strip
     @site[:description] = doc.at_xpath("//meta[@property='og:description']/@content")&.value
@@ -128,8 +141,11 @@ class Ficon
 
   def fetch_url(uri, redirect_limit = 5)
     uri = URI(uri) unless uri.is_a?(URI)
-
-    raise "Too many redirects" if redirect_limit <= 0
+    
+    if redirect_limit <= 0
+      @url_status = DEAD
+      raise "Too many redirects"
+    end
 
     Net::HTTP.start(uri.host, uri.port, use_ssl: uri.scheme == "https") do |http|
       http.read_timeout = 10
@@ -137,7 +153,10 @@ class Ficon
       request = Net::HTTP::Get.new(uri)
       request["User-Agent"] = @user_agent
       response = http.request(request)
-
+      
+      # Set status based on response
+      @url_status = classify_response_status(response)
+      
       case response
       when Net::HTTPRedirection
         location = response["location"]
@@ -149,11 +168,40 @@ class Ficon
       else
         @final_uri = Addressable::URI.parse(uri.to_s)
       end
-
+      
       response
     end
-  rescue Net::HTTPError, SocketError, Timeout::Error => e
+  rescue => e
+    @url_status = classify_exception_status(e)
     puts "Failed to fetch #{uri}: #{e.inspect}"
     nil
   end
+
+  def classify_response_status(response)
+    case response.code.to_i
+    when 200..299
+      ALIVE
+    when 404, 410
+      DEAD
+    when 401, 403, 429
+      BLOCKED
+    when 500..599
+      SICK
+    else
+      SICK
+    end
+  end
+
+  def classify_exception_status(exception)
+    case exception
+    when SocketError, Resolv::ResolutionError
+      DEAD  # DNS resolution failures
+    when Net::HTTPError, Timeout::Error, Errno::ECONNREFUSED
+      SICK  # Network issues worth retrying
+    when OpenSSL::SSL::SSLError
+      SICK  # SSL certificate errors
+    else
+      SICK  # Default to retryable for unknown errors
+    end
+  end
 end

data/test/ficon_test.rb

@@ -58,7 +58,7 @@ class FiconTest < Minitest::Test
   def test_custom_user_agent
     # Test default user agent
     ficon_default = Ficon.new('https://example.com')
-    assert_match(/^Ficon\/0\.2/, ficon_default.user_agent)
+    assert_match(/^FiconBot\/0\.\d+/, ficon_default.user_agent)
     
     # Test custom user agent
     custom_agent = 'MyApp/1.0 (Custom Bot)'

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: ficon
 version: !ruby/object:Gem::Version
-  version: '0.3'
+  version: '0.4'
 platform: ruby
 authors:
 - Dan Milne