webinspector 0.5.0 → 1.1.0

data/lib/web_inspector/meta.rb

@@ -1,15 +1,18 @@
+# frozen_string_literal: true
+
 module WebInspector
   class Meta
-  	def initialize(page)
+    def initialize(page)
       @page = page
     end
 
     def meta_tags
       {
-        'name'        => meta_tags_by('name'),
-        'http-equiv'  => meta_tags_by('http-equiv'),
-        'property'    => meta_tags_by('property'),
-        'charset'     => [charset_from_meta_charset]
+        'name' => meta_tags_by('name'),
+        'http-equiv' => meta_tags_by('http-equiv'),
+        'property' => meta_tags_by('property'),
+        'charset' => [charset_from_meta_charset],
+        'itemprop' => meta_tags_by('itemprop') # Add support for schema.org microdata
       }
     end
 
@@ -19,30 +22,48 @@ module WebInspector
 
     def meta
       meta_tag['name']
-          .merge(meta_tag['http-equiv'])
-          .merge(meta_tag['property'])
-          .merge('charset' => meta_tag['charset'])
+        .merge(meta_tag['http-equiv'])
+        .merge(meta_tag['property'])
+        .merge(meta_tag['itemprop'] || {})
+        .merge('charset' => meta_tag['charset'])
     end
 
     def charset
-      @charset ||= (charset_from_meta_charset || charset_from_meta_content_type)
+      @charset ||= charset_from_meta_charset || charset_from_meta_content_type || charset_from_header || 'utf-8'
     end
 
     private
 
     def charset_from_meta_charset
-      @page.css('meta[charset]')[0].attributes['charset'].value rescue nil
+      @page.css('meta[charset]')[0].attributes['charset'].value
+    rescue StandardError
+      nil
     end
 
     def charset_from_meta_content_type
-      @page.css("meta[http-equiv='Content-Type']")[0].attributes['content'].value.split(';')[1].split('=')[1] rescue nil
+      @page.css("meta[http-equiv='Content-Type']")[0].attributes['content'].value.split(';')[1].strip.split('=')[1]
+    rescue StandardError
+      nil
+    end
+
+    def charset_from_header
+      # Try to get charset from Content-Type header if available
+      nil
     end
 
-   	def meta_tags_by(attribute)
+    def meta_tags_by(attribute)
       hash = {}
       @page.css("meta[@#{attribute}]").map do |tag|
-        name    = tag.attributes[attribute].value.downcase rescue nil
-        content = tag.attributes['content'].value rescue nil
+        name = begin
+          tag.attributes[attribute].value.downcase
+        rescue StandardError
+          nil
+        end
+        content = begin
+          tag.attributes['content'].value
+        rescue StandardError
+          nil
+        end
 
         if name && content
           hash[name] ||= []
@@ -64,4 +85,4 @@ module WebInspector
       end
     end
   end
-end
+end

data/lib/web_inspector/page.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'nokogiri'
 require 'uri'
 require 'open-uri'
@@ -5,133 +7,316 @@ require 'open_uri_redirections'
 require 'faraday'
 require 'public_suffix'
 
+# Explicitly load Faraday::Retry if available
+begin
+  require 'faraday/retry'
+rescue LoadError
+  # Faraday retry is not available
+end
+
 require File.expand_path(File.join(File.dirname(__FILE__), 'inspector'))
 require File.expand_path(File.join(File.dirname(__FILE__), 'request'))
 
 module WebInspector
   class Page
-    attr_reader :url, :scheme, :host, :port, :title, :description, :body, :meta, :links, :domain_links, :domain_images, :images, :response
+    attr_reader :status_code
+
+    DEFAULT_TIMEOUT = 30
+    DEFAULT_RETRIES = 3
+    DEFAULT_USER_AGENT = -> { "WebInspector/#{WebInspector::VERSION} (+https://github.com/davidesantangelo/webinspector)" }
 
+    # Initialize a new WebInspector Page
+    #
+    # @param url [String] The URL to inspect
+    # @param options [Hash] Optional parameters
+    # @option options [Integer] :timeout Request timeout in seconds
+    # @option options [Integer] :retries Number of retries for failed requests
+    # @option options [Hash] :headers Custom HTTP headers
+    # @option options [Boolean] :allow_redirections Whether to follow redirects
+    # @option options [String] :user_agent Custom user agent
     def initialize(url, options = {})
       @url = url
       @options = options
+      @retries = options[:retries] || DEFAULT_RETRIES
+      @timeout = options[:timeout] || DEFAULT_TIMEOUT
+      @headers = options[:headers] || { 'User-Agent' => options[:user_agent] || DEFAULT_USER_AGENT.call }
+      @allow_redirections = options[:allow_redirections].nil? || options[:allow_redirections]
+
       @request = WebInspector::Request.new(url)
-      @inspector = WebInspector::Inspector.new(page)
-    end
 
-    def title
-      @inspector.title
+      begin
+        @inspector = WebInspector::Inspector.new(page)
+        @inspector.set_url(url, host)
+        @status_code = 200
+      rescue StandardError => e
+        @error = e
+        @status_code = e.respond_to?(:status_code) ? e.status_code : 500
+      end
     end
 
-    def description
-      @inspector.description
+    # Check if the page was successfully loaded
+    #
+    # @return [Boolean] true if the page was loaded, false otherwise
+    def success?
+      !@inspector.nil? && !@error
     end
 
-    def body
-      @inspector.body
-    end
-    
-    def links
-      @inspector.links
+    # Get the error message if any
+    #
+    # @return [String, nil] The error message or nil if no error
+    def error_message
+      @error&.message
     end
 
-    def images
-      @inspector.images
-    end
+    # Delegate methods to inspector
+    %i[title description body links images meta javascripts stylesheets language structured_data microdata
+       tag_count].each do |method|
+      define_method(method) do
+        return nil unless success?
 
-    def meta
-      @inspector.meta
+        @inspector.send(method)
+      end
     end
 
+    # Special case for find method that takes arguments
     def find(words)
+      return nil unless success?
+
       @inspector.find(words)
     end
 
-    def url
-      @request.url
+    # Delegate methods to request
+    %i[url host domain scheme port].each do |method|
+      define_method(method) do
+        @request.send(method)
+      end
     end
 
-    def host
-      @request.host
-    end
+    # Get the favicon URL if available
+    #
+    # @return [String, nil] The favicon URL or nil if not found
+    def favicon
+      return @favicon if defined?(@favicon)
 
-    def domain
-      @request.domain
-    end
+      return nil unless success?
 
-    def scheme
-      @request.scheme
-    end
+      @favicon = begin
+        # Try multiple approaches to find favicon
+
+        # 1. Look for standard favicon link tags
+        favicon_link = @inspector.page.css("link[rel='shortcut icon'], link[rel='icon'], link[rel='apple-touch-icon']").first
+        if favicon_link && favicon_link['href']
+          begin
+            return URI.join(url, favicon_link['href']).to_s
+          rescue URI::InvalidURIError
+            # Try next method
+          end
+        end
 
-    def port
-      @request.port
+        # 2. Try the default location /favicon.ico
+        "#{scheme}://#{host}/favicon.ico"
+      rescue StandardError
+        nil
+      end
     end
 
     def domain_links(u = domain)
+      return [] unless success?
+
       @inspector.domain_links(u, host)
     end
 
     def domain_images(u = domain)
+      return [] unless success?
+
       @inspector.domain_images(u, host)
     end
 
+    # Get information about the page's security
+    # @return [Hash] Security information
+    def security_info
+      return @security_info if defined?(@security_info)
+
+      @security_info = {
+        secure: scheme == 'https',
+        hsts: response&.headers && response.headers['strict-transport-security'] ? true : false,
+        content_security_policy: response&.headers && response.headers['content-security-policy'] ? true : false
+      }
+
+      # Extract SSL/TLS info if available and using HTTPS
+      if scheme == 'https' && response&.env&.response_headers
+        @security_info[:ssl_version] = response.env[:ssl_version]
+        @security_info[:cipher_suite] = response.env[:cipher_suite]
+      end
+
+      @security_info
+    end
+
+    # Get the content type of the page
+    # @return [String, nil] Content type
+    def content_type
+      response&.headers && response.headers['content-type']
+    end
+
+    # Get the size of the page in bytes
+    # @return [Integer, nil] Size in bytes
+    def size
+      return @size if defined?(@size)
+
+      @size = if response&.headers && response.headers['content-length']
+                response.headers['content-length'].to_i
+              elsif response&.body
+                response.body.bytesize
+              end
+    end
+
+    # Get the load time of the page in seconds
+    # @return [Float, nil] Load time in seconds
+    attr_reader :load_time
+
+    # Get all JSON-LD structured data as a hash
+    # @return [Array<Hash>] Structured data
+    def json_ld
+      structured_data
+    end
+
+    # Get a hash of all technologies detected on the page
+    # @return [Hash] Detected technologies
+    def technologies
+      techs = {}
+      js_files = javascripts || []
+      css_files = stylesheets || []
+      page_body = body || ''
+      page_meta = meta || {}
+      response_headers = response&.headers || {}
+
+      # Frameworks and Libraries
+      techs[:jquery] = true if js_files.any? { |js| js.include?('jquery') } || page_body.include?('jQuery')
+      techs[:react] = true if page_body.include?('data-reactroot') || js_files.any? { |js| js.include?('react') }
+      techs[:vue] = true if page_body.include?('data-v-app') || js_files.any? { |js| js.include?('vue') }
+      techs[:angular] = true if page_body.include?('ng-version') || js_files.any? { |js| js.include?('angular') }
+      techs[:bootstrap] = true if css_files.any? do |css|
+        css.include?('bootstrap')
+      end || page_body.include?('class="container"')
+      if response_headers['x-powered-by']&.include?('Rails') || response_headers.key?('x-rails-env')
+        techs[:rails] =
+          true
+      end
+      techs[:php] = true if response_headers['x-powered-by']&.include?('PHP')
+
+      # CMS
+      techs[:wordpress] = true if page_meta['generator']&.include?('WordPress') || page_body.include?('/wp-content/')
+      techs[:shopify] = true if page_body.include?('Shopify.shop')
+
+      # Analytics
+      techs[:google_analytics] = true if js_files.any? { |js| js.include?('google-analytics.com') }
+
+      # Server
+      server = response_headers['server']
+      if server
+        techs[:server] = server
+        techs[:nginx] = true if server.include?('nginx')
+        techs[:apache] = true if server.include?('Apache')
+        techs[:iis] = true if server.include?('IIS')
+        techs[:express] = true if response_headers['x-powered-by']&.include?('Express')
+      end
+
+      techs
+    end
+
+    # Get full JSON representation of the page with all new data
+    # @return [Hash] JSON representation of the page
     def to_hash
       {
-        'url'           => url,
-        'scheme'        => scheme,
-        'host'          => host,
-        'port'          => port,
-        'title'         => title,
-        'description'  	=> description,
-        'meta'  				=> meta,
-        'links'					=> links,
-        'images'				=> images,
-        'response'      => { 'status'  => response.status,
-                             'headers' => response.headers }
+        'url' => url,
+        'scheme' => scheme,
+        'host' => host,
+        'port' => port,
+        'title' => title,
+        'description' => description,
+        'meta' => meta,
+        'links' => links,
+        'images' => images,
+        'javascripts' => javascripts,
+        'stylesheets' => stylesheets,
+        'favicon' => favicon,
+        'language' => language,
+        'structured_data' => structured_data,
+        'microdata' => microdata,
+        'security_info' => security_info,
+        'content_type' => content_type,
+        'size' => size,
+        'load_time' => load_time,
+        'technologies' => technologies,
+        'tag_count' => tag_count,
+        'response' => {
+          'status' => status_code,
+          'headers' => response&.headers || {},
+          'success' => success?
+        },
+        'error' => error_message
       }
     end
 
     def response
       @response ||= fetch
-    rescue Faraday::TimeoutError, Faraday::Error::ConnectionFailed, RuntimeError, URI::InvalidURIError => e
+    rescue StandardError => e
+      @error = e
       nil
     end
 
     private
-    
+
     def fetch
-      session = Faraday.new(:url => url) do |faraday|
-        faraday.request :retry, max: @retries
+      start_time = Time.now
 
+      session = Faraday.new(url: url) do |faraday|
+        # Configure retries based on available middleware
+        faraday.request :retry, { max: @retries } if defined?(Faraday::Retry)
+
+        # Configure redirect handling
         if @allow_redirections
-          faraday.use FaradayMiddleware::FollowRedirects, limit: 10
-          faraday.use :cookie_jar
+          begin
+            faraday.use FaradayMiddleware::FollowRedirects, limit: 10
+            faraday.use :cookie_jar
+          rescue NameError, NoMethodError
+            # Continue without middleware if not available
+          end
         end
 
-        faraday.headers.merge!(@headers || {})
+        faraday.headers.merge!(@headers)
         faraday.adapter :net_http
       end
 
-      response = session.get do |req|
-        req.options.timeout      = @connection_timeout
-        req.options.open_timeout = @read_timeout
-      end
+      # Manual retry mechanism as a backup
+      retries = 0
 
-      @url = response.env.url.to_s
+      begin
+        response = session.get do |req|
+          req.options.timeout = @timeout
+          req.options.open_timeout = @timeout
+        end
 
-      response
+        @url = response.env.url.to_s
+        @load_time = Time.now - start_time
+        response
+      rescue Faraday::TimeoutError, Faraday::ConnectionFailed => e
+        retries += 1
+        retry if retries <= @retries
+        raise e
+      end
     end
 
     def with_default_scheme(request)
-      request.url && request.scheme.nil? ? 'http://' + request.url : request.url
-    end
-
-    def default_user_agent
-      "WebInspector/#{WebInspector::VERSION} (+https://github.com/davidesantangelo/webinspector)"
+      request.url && request.scheme.nil? ? "http://#{request.url}" : request.url
     end
 
     def page
-      Nokogiri::HTML(open(with_default_scheme(@request), allow_redirections: :safe))
+      # Use URI.open instead of open for Ruby 3.0+ compatibility
+      Nokogiri::HTML(URI.open(with_default_scheme(@request),
+                              allow_redirections: :safe,
+                              read_timeout: @timeout,
+                              'User-Agent' => @headers['User-Agent']))
     end
   end
 end

data/lib/web_inspector/request.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'addressable/uri'
 
 module WebInspector
@@ -13,7 +15,7 @@ module WebInspector
     def host
       uri.host
     end
-    
+
     def domain
       suffix_domain
     end
@@ -24,23 +26,23 @@ module WebInspector
 
     def port
       URI(normalized_uri).port
-    end 
+    end
 
     private
-    
+
     def suffix_domain
       return @domain if @domain
-      
+
       begin
         @domain = PublicSuffix.parse(host).domain
-      rescue URI::InvalidURIError, PublicSuffix::DomainInvalid => e
+      rescue URI::InvalidURIError, PublicSuffix::DomainInvalid
         @domain = ''
       end
     end
-    
+
     def uri
       Addressable::URI.parse(@url)
-    rescue Addressable::URI::InvalidURIError => e
+    rescue Addressable::URI::InvalidURIError
       nil
     end
 
@@ -48,4 +50,4 @@ module WebInspector
       uri.normalize.to_s
     end
   end
-end
+end

data/lib/web_inspector/version.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module WebInspector
-  VERSION = "0.5.0"
+  VERSION = '1.1.0'
 end

data/lib/web_inspector.rb

@@ -1,10 +1,12 @@
+# frozen_string_literal: true
+
 require File.expand_path(File.join(File.dirname(__FILE__), 'web_inspector/page'))
 require File.expand_path(File.join(File.dirname(__FILE__), 'web_inspector/version'))
 
 module WebInspector
-  extend self
+  module_function
 
   def new(url, options = {})
     Page.new(url, options)
   end
-end
+end

data/lib/webinspector.rb

@@ -1 +1,3 @@
-require File.expand_path(File.join(File.dirname(__FILE__), './web_inspector'))
+# frozen_string_literal: true
+
+require File.expand_path(File.join(File.dirname(__FILE__), './web_inspector'))

data/webinspector.gemspec

@@ -1,38 +1,45 @@
-# coding: utf-8
-lib = File.expand_path('../lib', __FILE__)
+# frozen_string_literal: true
+
+lib = File.expand_path('lib', __dir__)
 $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
-require File.expand_path('../lib/web_inspector/version', __FILE__)
+require File.expand_path('lib/web_inspector/version', __dir__)
 
 Gem::Specification.new do |spec|
-  spec.name          = "webinspector"
+  spec.name          = 'webinspector'
   spec.version       = WebInspector::VERSION
-  spec.authors       = ["Davide Santangelo"]
-  spec.email         = ["davide.santangelo@gmail.com"]
+  spec.authors       = ['Davide Santangelo']
+  spec.email         = ['davide.santangelo@gmail.com']
 
-  spec.summary       = %q{Ruby gem to inspect completely a web page.}
-  spec.description   = %q{Ruby gem to inspect completely a web page. It scrapes a given URL, and returns you its meta, links, images and more.}
-  spec.homepage      = ""
-  spec.license       = "MIT"
+  spec.summary       = 'Ruby gem to inspect completely a web page.'
+  spec.description   = 'Ruby gem to inspect completely a web page. It scrapes a given URL, and returns you its meta, links, images and more.'
+  spec.homepage      = 'https://github.com/davidesantangelo/webinspector'
+  spec.license       = 'MIT'
 
   spec.files         = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
-  spec.bindir        = "exe"
+  spec.bindir        = 'exe'
   spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
-  spec.require_paths = ["lib"]
-
-  spec.add_development_dependency "bundler", "~> 1.8"
-  spec.add_development_dependency "rake", "~> 10.0"
+  spec.require_paths = ['lib']
+  spec.metadata      = {
+    'source_code_uri' => 'https://github.com/davidesantangelo/webinspector',
+    'bug_tracker_uri' => 'https://github.com/davidesantangelo/webinspector/issues'
+  }
 
-  spec.add_development_dependency 'rspec'
-  spec.add_development_dependency "vcr"
-  spec.add_development_dependency "typhoeus"
+  spec.required_ruby_version = '>= 3.0.0'
 
-  spec.required_ruby_version = ">= 1.9.3"
+  spec.add_development_dependency 'rake', '~> 13.0'
+  spec.add_development_dependency 'rspec', '~> 3.12'
+  spec.add_development_dependency 'rubocop', '~> 1.50'
+  spec.add_development_dependency 'vcr', '~> 6.1'
+  spec.add_development_dependency 'webmock', '~> 3.18'
 
-  spec.add_dependency "faraday"
-  spec.add_dependency "json"
-  spec.add_dependency "addressable"
-  spec.add_dependency "nokogiri"
-  spec.add_dependency "open_uri_redirections"
-  spec.add_dependency "openurl"
-  spec.add_dependency "public_suffix"
+  spec.add_dependency 'addressable', '~> 2.8'
+  spec.add_dependency 'faraday', '~> 2.7'
+  spec.add_dependency 'faraday-cookie_jar', '~> 0.0.7'
+  spec.add_dependency 'faraday-follow_redirects', '~> 0.3'
+  spec.add_dependency 'faraday-retry', '~> 2.1'
+  spec.add_dependency 'json', '~> 2.6'
+  spec.add_dependency 'nokogiri', '~> 1.14'
+  spec.add_dependency 'open_uri_redirections', '~> 0.2'
+  spec.add_dependency 'openurl', '~> 1.0'
+  spec.add_dependency 'public_suffix', '~> 5.0'
 end