webinspector 0.5.0 → 1.0.0

data/lib/web_inspector/page.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'nokogiri'
 require 'uri'
 require 'open-uri'
@@ -5,133 +7,211 @@ require 'open_uri_redirections'
 require 'faraday'
 require 'public_suffix'
 
+# Explicitly load Faraday::Retry if available
+begin
+  require 'faraday/retry'
+rescue LoadError
+  # Faraday retry is not available
+end
+
 require File.expand_path(File.join(File.dirname(__FILE__), 'inspector'))
 require File.expand_path(File.join(File.dirname(__FILE__), 'request'))
 
 module WebInspector
   class Page
-    attr_reader :url, :scheme, :host, :port, :title, :description, :body, :meta, :links, :domain_links, :domain_images, :images, :response
-
+    attr_reader :url, :scheme, :host, :port, :title, :description, :body, :meta, :links,
+                :domain_links, :domain_images, :images, :response, :status_code, :favicon
+
+    DEFAULT_TIMEOUT = 30
+    DEFAULT_RETRIES = 3
+    DEFAULT_USER_AGENT = -> { "WebInspector/#{WebInspector::VERSION} (+https://github.com/davidesantangelo/webinspector)" }
+
+    # Initialize a new WebInspector Page
+    #
+    # @param url [String] The URL to inspect
+    # @param options [Hash] Optional parameters
+    # @option options [Integer] :timeout Request timeout in seconds
+    # @option options [Integer] :retries Number of retries for failed requests
+    # @option options [Hash] :headers Custom HTTP headers
+    # @option options [Boolean] :allow_redirections Whether to follow redirects
+    # @option options [String] :user_agent Custom user agent
     def initialize(url, options = {})
       @url = url
       @options = options
+      @retries = options[:retries] || DEFAULT_RETRIES
+      @timeout = options[:timeout] || DEFAULT_TIMEOUT
+      @headers = options[:headers] || { 'User-Agent' => options[:user_agent] || DEFAULT_USER_AGENT.call }
+      @allow_redirections = options[:allow_redirections].nil? || options[:allow_redirections]
+
       @request = WebInspector::Request.new(url)
-      @inspector = WebInspector::Inspector.new(page)
-    end
 
-    def title
-      @inspector.title
+      begin
+        @inspector = WebInspector::Inspector.new(page)
+        @inspector.set_url(url, host)
+        @status_code = 200
+      rescue StandardError => e
+        @error = e
+        @status_code = e.respond_to?(:status_code) ? e.status_code : 500
+      end
     end
 
-    def description
-      @inspector.description
+    # Check if the page was successfully loaded
+    #
+    # @return [Boolean] true if the page was loaded, false otherwise
+    def success?
+      !@inspector.nil? && !@error
     end
 
-    def body
-      @inspector.body
-    end
-    
-    def links
-      @inspector.links
+    # Get the error message if any
+    #
+    # @return [String, nil] The error message or nil if no error
+    def error_message
+      @error&.message
     end
 
-    def images
-      @inspector.images
-    end
+    # Delegate methods to inspector
+    %i[title description body links images meta].each do |method|
+      define_method(method) do
+        return nil unless success?
 
-    def meta
-      @inspector.meta
+        @inspector.send(method)
+      end
     end
 
+    # Special case for find method that takes arguments
     def find(words)
+      return nil unless success?
+
       @inspector.find(words)
     end
 
-    def url
-      @request.url
+    # Delegate methods to request
+    %i[url host domain scheme port].each do |method|
+      define_method(method) do
+        @request.send(method)
+      end
     end
 
-    def host
-      @request.host
-    end
+    # Get the favicon URL if available
+    #
+    # @return [String, nil] The favicon URL or nil if not found
+    def favicon
+      return @favicon if defined?(@favicon)
 
-    def domain
-      @request.domain
-    end
+      return nil unless success?
 
-    def scheme
-      @request.scheme
-    end
+      @favicon = begin
+        # Try multiple approaches to find favicon
+
+        # 1. Look for standard favicon link tags
+        favicon_link = @inspector.page.css("link[rel='shortcut icon'], link[rel='icon'], link[rel='apple-touch-icon']").first
+        if favicon_link && favicon_link['href']
+          begin
+            return URI.join(url, favicon_link['href']).to_s
+          rescue URI::InvalidURIError
+            # Try next method
+          end
+        end
 
-    def port
-      @request.port
+        # 2. Try the default location /favicon.ico
+        "#{scheme}://#{host}/favicon.ico"
+      rescue StandardError
+        nil
+      end
     end
 
     def domain_links(u = domain)
+      return [] unless success?
+
       @inspector.domain_links(u, host)
     end
 
     def domain_images(u = domain)
+      return [] unless success?
+
       @inspector.domain_images(u, host)
     end
 
+    # Get full JSON representation of the page
+    #
+    # @return [Hash] JSON representation of the page
     def to_hash
       {
-        'url'           => url,
-        'scheme'        => scheme,
-        'host'          => host,
-        'port'          => port,
-        'title'         => title,
-        'description'  	=> description,
-        'meta'  				=> meta,
-        'links'					=> links,
-        'images'				=> images,
-        'response'      => { 'status'  => response.status,
-                             'headers' => response.headers }
+        'url' => url,
+        'scheme' => scheme,
+        'host' => host,
+        'port' => port,
+        'title' => title,
+        'description' => description,
+        'meta' => meta,
+        'links' => links,
+        'images' => images,
+        'favicon' => favicon,
+        'response' => {
+          'status' => status_code,
+          'headers' => response&.headers || {},
+          'success' => success?
+        },
+        'error' => error_message
       }
     end
 
     def response
       @response ||= fetch
-    rescue Faraday::TimeoutError, Faraday::Error::ConnectionFailed, RuntimeError, URI::InvalidURIError => e
+    rescue StandardError => e
+      @error = e
       nil
     end
 
     private
-    
+
     def fetch
-      session = Faraday.new(:url => url) do |faraday|
-        faraday.request :retry, max: @retries
+      session = Faraday.new(url: url) do |faraday|
+        # Configure retries based on available middleware
+        faraday.request :retry, { max: @retries } if defined?(Faraday::Retry)
 
+        # Configure redirect handling
         if @allow_redirections
-          faraday.use FaradayMiddleware::FollowRedirects, limit: 10
-          faraday.use :cookie_jar
+          begin
+            faraday.use FaradayMiddleware::FollowRedirects, limit: 10
+            faraday.use :cookie_jar
+          rescue NameError, NoMethodError
+            # Continue without middleware if not available
+          end
         end
 
-        faraday.headers.merge!(@headers || {})
+        faraday.headers.merge!(@headers)
         faraday.adapter :net_http
       end
 
-      response = session.get do |req|
-        req.options.timeout      = @connection_timeout
-        req.options.open_timeout = @read_timeout
-      end
+      # Manual retry mechanism as a backup
+      retries = 0
 
-      @url = response.env.url.to_s
+      begin
+        response = session.get do |req|
+          req.options.timeout = @timeout
+          req.options.open_timeout = @timeout
+        end
 
-      response
+        @url = response.env.url.to_s
+        response
+      rescue Faraday::TimeoutError, Faraday::ConnectionFailed => e
+        retries += 1
+        retry if retries <= @retries
+        raise e
+      end
     end
 
     def with_default_scheme(request)
-      request.url && request.scheme.nil? ? 'http://' + request.url : request.url
-    end
-
-    def default_user_agent
-      "WebInspector/#{WebInspector::VERSION} (+https://github.com/davidesantangelo/webinspector)"
+      request.url && request.scheme.nil? ? "http://#{request.url}" : request.url
     end
 
     def page
-      Nokogiri::HTML(open(with_default_scheme(@request), allow_redirections: :safe))
+      # Use URI.open instead of open for Ruby 3.0+ compatibility
+      Nokogiri::HTML(URI.open(with_default_scheme(@request),
+                              allow_redirections: :safe,
+                              read_timeout: @timeout,
+                              'User-Agent' => @headers['User-Agent']))
     end
   end
 end

data/lib/web_inspector/request.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'addressable/uri'
 
 module WebInspector
@@ -13,7 +15,7 @@ module WebInspector
     def host
       uri.host
     end
-    
+
     def domain
       suffix_domain
     end
@@ -24,23 +26,23 @@ module WebInspector
 
     def port
       URI(normalized_uri).port
-    end 
+    end
 
     private
-    
+
     def suffix_domain
       return @domain if @domain
-      
+
       begin
         @domain = PublicSuffix.parse(host).domain
-      rescue URI::InvalidURIError, PublicSuffix::DomainInvalid => e
+      rescue URI::InvalidURIError, PublicSuffix::DomainInvalid
         @domain = ''
       end
     end
-    
+
     def uri
       Addressable::URI.parse(@url)
-    rescue Addressable::URI::InvalidURIError => e
+    rescue Addressable::URI::InvalidURIError
       nil
     end
 
@@ -48,4 +50,4 @@ module WebInspector
       uri.normalize.to_s
     end
   end
-end
+end

data/lib/web_inspector/version.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module WebInspector
-  VERSION = "0.5.0"
+  VERSION = '1.0.0'
 end

data/lib/web_inspector.rb

@@ -1,10 +1,12 @@
+# frozen_string_literal: true
+
 require File.expand_path(File.join(File.dirname(__FILE__), 'web_inspector/page'))
 require File.expand_path(File.join(File.dirname(__FILE__), 'web_inspector/version'))
 
 module WebInspector
-  extend self
+  module_function
 
   def new(url, options = {})
     Page.new(url, options)
   end
-end
+end

data/lib/webinspector.rb

@@ -1 +1,3 @@
-require File.expand_path(File.join(File.dirname(__FILE__), './web_inspector'))
+# frozen_string_literal: true
+
+require File.expand_path(File.join(File.dirname(__FILE__), './web_inspector'))

data/webinspector.gemspec

@@ -1,38 +1,45 @@
-# coding: utf-8
-lib = File.expand_path('../lib', __FILE__)
+# frozen_string_literal: true
+
+lib = File.expand_path('lib', __dir__)
 $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
-require File.expand_path('../lib/web_inspector/version', __FILE__)
+require File.expand_path('lib/web_inspector/version', __dir__)
 
 Gem::Specification.new do |spec|
-  spec.name          = "webinspector"
+  spec.name          = 'webinspector'
   spec.version       = WebInspector::VERSION
-  spec.authors       = ["Davide Santangelo"]
-  spec.email         = ["davide.santangelo@gmail.com"]
+  spec.authors       = ['Davide Santangelo']
+  spec.email         = ['davide.santangelo@gmail.com']
 
-  spec.summary       = %q{Ruby gem to inspect completely a web page.}
-  spec.description   = %q{Ruby gem to inspect completely a web page. It scrapes a given URL, and returns you its meta, links, images and more.}
-  spec.homepage      = ""
-  spec.license       = "MIT"
+  spec.summary       = 'Ruby gem to inspect completely a web page.'
+  spec.description   = 'Ruby gem to inspect completely a web page. It scrapes a given URL, and returns you its meta, links, images and more.'
+  spec.homepage      = 'https://github.com/davidesantangelo/webinspector'
+  spec.license       = 'MIT'
 
   spec.files         = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
-  spec.bindir        = "exe"
+  spec.bindir        = 'exe'
   spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
-  spec.require_paths = ["lib"]
-
-  spec.add_development_dependency "bundler", "~> 1.8"
-  spec.add_development_dependency "rake", "~> 10.0"
+  spec.require_paths = ['lib']
+  spec.metadata      = {
+    'source_code_uri' => 'https://github.com/davidesantangelo/webinspector',
+    'bug_tracker_uri' => 'https://github.com/davidesantangelo/webinspector/issues'
+  }
 
-  spec.add_development_dependency 'rspec'
-  spec.add_development_dependency "vcr"
-  spec.add_development_dependency "typhoeus"
+  spec.required_ruby_version = '>= 3.0.0'
 
-  spec.required_ruby_version = ">= 1.9.3"
+  spec.add_development_dependency 'rake', '~> 13.0'
+  spec.add_development_dependency 'rspec', '~> 3.12'
+  spec.add_development_dependency 'rubocop', '~> 1.50'
+  spec.add_development_dependency 'vcr', '~> 6.1'
+  spec.add_development_dependency 'webmock', '~> 3.18'
 
-  spec.add_dependency "faraday"
-  spec.add_dependency "json"
-  spec.add_dependency "addressable"
-  spec.add_dependency "nokogiri"
-  spec.add_dependency "open_uri_redirections"
-  spec.add_dependency "openurl"
-  spec.add_dependency "public_suffix"
+  spec.add_dependency 'addressable', '~> 2.8'
+  spec.add_dependency 'faraday', '~> 2.7'
+  spec.add_dependency 'faraday-cookie_jar', '~> 0.0.7'
+  spec.add_dependency 'faraday-follow_redirects', '~> 0.3'
+  spec.add_dependency 'faraday-retry', '~> 2.1'
+  spec.add_dependency 'json', '~> 2.6'
+  spec.add_dependency 'nokogiri', '~> 1.14'
+  spec.add_dependency 'open_uri_redirections', '~> 0.2'
+  spec.add_dependency 'openurl', '~> 1.0'
+  spec.add_dependency 'public_suffix', '~> 5.0'
 end