webinspector 1.0.0 → 1.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 0413d3ff948ab6efff6a1cbe8a7844287149ad06f09353655e6cb208968f9481
-  data.tar.gz: 152b950595afb57adc522da24c6959f71d160ba903b3d01ce6ee5f6a8b4d81d2
+  metadata.gz: 7ef69ac6245db062331d4f0b269d1f389cb9e730e4ad39e51d75236f30e0b6c9
+  data.tar.gz: b37d5b30a01ea6b0e74553edb9944acdeac4d5ce2808e319798b8d0d3a9f5390
 SHA512:
-  metadata.gz: c6230493b59a0d23585be729ec98706cfdbd6852e2de2d65db83d1638f85110369d41f2275b8e0aa09b58008d53924d036840ce63523d9004f19275999be90f8
-  data.tar.gz: dad518b0b04c1e341c14c29438ebcf84f4602bf394254a4c61382ad79ce53dae2ac92f138f331d5bf089d2220011f14b0cde0a79608a91177b2bda2ab1773a96
+  metadata.gz: '098f7c7f19f8bfe8b8f3b4977cac590e9c107cb8864128e4abbb3b47327e34743844fcde2d5bf6e3038e39e7271d565dbd19157b703ea041f31e4c3fbc84aa47'
+  data.tar.gz: 5fdeaa74861245796f15bf70ba5fcc00e7e7eb2f0c1192a60395a119b24ea1c72b579a24296cb66c1fd77cc7c4cfff8b2505d321a2ef74dd7b5618b84e69ae5c

data/README.md

@@ -4,7 +4,6 @@ Ruby gem to inspect web pages. It scrapes a given URL and returns its title, des
 
 <a href="https://codeclimate.com/github/davidesantangelo/webinspector"><img src="https://codeclimate.com/github/davidesantangelo/webinspector/badges/gpa.svg" /></a>
 
-
 ## Installation
 
 Add this line to your application's Gemfile:
@@ -34,7 +33,7 @@ page = WebInspector.new('http://example.com')
 ```ruby
 page = WebInspector.new('http://example.com', {
   timeout: 30,                         # Request timeout in seconds (default: 30)
-  retries: 3,                          # Number of retries (default: 3) 
+  retries: 3,                          # Number of retries (default: 3)
   headers: {'User-Agent': 'Custom UA'} # Custom HTTP headers
 })
 ```
@@ -86,16 +85,125 @@ page.domain_images('example.com') # returns only images hosted on example.com
 page.find(["ruby", "rails"]) # returns [{"ruby"=>3}, {"rails"=>1}]
 ```
 
+#### JavaScript and Stylesheets
+
+```ruby
+page.javascripts  # array of all JavaScript files (absolute URLs)
+page.stylesheets  # array of all CSS stylesheets (absolute URLs)
+```
+
+#### Language Detection
+
+```ruby
+page.language  # detected language code (e.g., "en", "es", "fr")
+```
+
+#### Structured Data
+
+```ruby
+page.structured_data  # array of JSON-LD structured data objects
+page.microdata        # array of microdata items
+page.json_ld          # alias for structured_data
+```
+
+#### Security Information
+
+```ruby
+page.security_info  # hash with security details: { secure: true, hsts: true, ... }
+```
+
+#### Performance Metrics
+
+```ruby
+page.load_time  # page load time in seconds
+page.size       # page size in bytes
+```
+
+#### Content Type
+
+```ruby
+page.content_type  # content type header (e.g., "text/html; charset=utf-8")
+```
+
+#### Technology Detection
+
+```ruby
+page.technologies  # hash of detected technologies: { jquery: true, react: true, ... }
+```
+
+#### HTML Tag Statistics
+
+```ruby
+page.tag_count  # hash with counts of each HTML tag: { "div" => 45, "p" => 12, ... }
+```
+
+#### RSS/Atom Feeds
+
+```ruby
+page.feeds  # array of RSS/Atom feed URLs found on the page
+```
+
+#### Social Media Links
+
+```ruby
+page.social_links  # hash of social media profiles: { facebook: "url", twitter: "url", ... }
+```
+
+#### Robots.txt and Sitemap
+
+```ruby
+page.robots_txt_url  # URL to robots.txt
+page.sitemap_url     # array of sitemap URLs
+```
+
+#### CMS Detection
+
+```ruby
+page.cms_info  # hash with CMS details: { name: "WordPress", version: "6.0", themes: [...], plugins: [...] }
+```
+
+#### Accessibility Score
+
+```ruby
+page.accessibility_score  # hash with score (0-100) and details: { score: 85, details: [...] }
+```
+
+#### Mobile-Friendly Check
+
+```ruby
+page.mobile_friendly?  # true if the page has viewport meta tag and responsive CSS
+```
+
 ### Export all data to JSON
 
 ```ruby
 page.to_hash # returns a hash with all page data
 ```
 
+## Changelog
+
+### Version 1.2.0
+
+**New Features:**
+
+- RSS/Atom feed detection with `feeds` method
+- Social media profile extraction with `social_links` method
+- CMS detection and information with `cms_info` method (WordPress, Drupal, Joomla, Shopify, Wix, Squarespace)
+- Accessibility scoring with `accessibility_score` method
+- Mobile-friendly detection with `mobile_friendly?` method
+- Robots.txt and sitemap URL detection with `robots_txt_url` and `sitemap_url` methods
+
+**Improvements:**
+
+- Enhanced `Request` module with `valid?` and `ssl?` methods for better URL validation
+- Improved `Meta` module with author and publisher extraction
+- Better error handling across all modules
+- Performance improvements with internal caching
+
 ## Contributors
 
-  * Steven Shelby ([@stevenshelby](https://github.com/stevenshelby))
-  * Sam Nissen ([@samnissen](https://github.com/samnissen))
+- Steven Shelby ([@stevenshelby](https://github.com/stevenshelby))
+- Sam Nissen ([@samnissen](https://github.com/samnissen))
 
 ## License
 

data/lib/web_inspector/inspector.rb

@@ -71,26 +71,7 @@ module WebInspector
     # @return [Array<String>] Filtered links
     def domain_links(user_domain, host = nil)
       @host ||= host
-
-      return [] if links.empty?
-
-      # Handle nil user_domain
-      user_domain = @host.to_s if user_domain.nil? || user_domain.empty?
-
-      # Normalize domain for comparison
-      user_domain = user_domain.to_s.downcase.gsub(/\s+/, '')
-      user_domain = user_domain.sub(/^www\./, '') # Remove www prefix for comparison
-
-      links.select do |link|
-        uri = URI.parse(link.to_s)
-        next false unless uri.host # Skip URLs without hosts
-
-        uri_host = uri.host.to_s.downcase
-        uri_host = uri_host.sub(/^www\./, '') # Remove www prefix for comparison
-        uri_host.include?(user_domain)
-      rescue URI::InvalidURIError, NoMethodError
-        false
-      end
+      filter_by_domain(links, user_domain)
     end
 
     # Get all images from the page
@@ -122,25 +103,325 @@ module WebInspector
     # @return [Array<String>] Filtered images
     def domain_images(user_domain, host = nil)
       @host ||= host
+      filter_by_domain(images, user_domain)
+    end
 
-      return [] if images.empty?
+    # Get all JavaScript files used by the page
+    # @return [Array<String>] Array of JavaScript file URLs
+    def javascripts
+      @javascripts ||= begin
+        scripts = []
+        @page.css('script[src]').each do |script|
+          src = script[:src]
+          next unless src
 
-      # Handle nil user_domain
-      user_domain = @host.to_s if user_domain.nil? || user_domain.empty?
+          # Clean and normalize URL
+          src = src.strip
 
-      # Normalize domain for comparison
-      user_domain = user_domain.to_s.downcase.gsub(/\s+/, '')
-      user_domain = user_domain.sub(/^www\./, '') # Remove www prefix for comparison
+          begin
+            absolute_url = make_absolute_url(src)
+            scripts << absolute_url if absolute_url
+          rescue URI::InvalidURIError, URI::BadURIError
+            # Skip invalid URLs
+          end
+        end
+        scripts.uniq.compact
+      end
+    end
 
-      images.select do |img|
-        uri = URI.parse(img.to_s)
-        next false unless uri.host # Skip URLs without hosts
+    # Get stylesheets used by the page
+    # @return [Array<String>] Array of CSS file URLs
+    def stylesheets
+      @stylesheets ||= begin
+        styles = []
+        @page.css('link[rel="stylesheet"]').each do |style|
+          href = style[:href]
+          next unless href
 
-        uri_host = uri.host.to_s.downcase
-        uri_host = uri_host.sub(/^www\./, '') # Remove www prefix for comparison
-        uri_host.include?(user_domain)
-      rescue URI::InvalidURIError, NoMethodError
-        false
+          # Clean and normalize URL
+          href = href.strip
+
+          begin
+            absolute_url = make_absolute_url(href)
+            styles << absolute_url if absolute_url
+          rescue URI::InvalidURIError, URI::BadURIError
+            # Skip invalid URLs
+          end
+        end
+        styles.uniq.compact
+      end
+    end
+
+    # Detect the page language
+    # @return [String, nil] Language code if detected, nil otherwise
+    def language
+      # Check for html lang attribute first
+      html_tag = @page.at('html')
+      return html_tag['lang'] if html_tag && html_tag['lang'] && !html_tag['lang'].empty?
+
+      # Then check for language meta tag
+      lang_meta = @meta['content-language']
+      return lang_meta if lang_meta && !lang_meta.empty?
+
+      # Fallback to inspecting content headers if available
+      nil
+    end
+
+    # Extract structured data (JSON-LD) from the page
+    # @return [Array<Hash>] Array of structured data objects
+    def structured_data
+      @structured_data ||= begin
+        data = []
+        @page.css('script[type="application/ld+json"]').each do |script|
+          parsed = JSON.parse(script.text)
+          data << parsed if parsed
+        rescue JSON::ParserError
+          # Skip invalid JSON
+        end
+        data
+      end
+    end
+
+    # Extract microdata from the page
+    # @return [Array<Hash>] Array of microdata items
+    def microdata
+      @microdata ||= begin
+        items = []
+        @page.css('[itemscope]').each do |scope|
+          item = { type: scope['itemtype'] }
+          properties = {}
+
+          scope.css('[itemprop]').each do |prop|
+            name = prop['itemprop']
+            # Extract value based on tag
+            value = case prop.name.downcase
+                    when 'meta'
+                      prop['content']
+                    when 'img', 'audio', 'embed', 'iframe', 'source', 'track', 'video'
+                      make_absolute_url(prop['src'])
+                    when 'a', 'area', 'link'
+                      make_absolute_url(prop['href'])
+                    when 'time'
+                      prop['datetime'] || prop.text.strip
+                    else
+                      prop.text.strip
+                    end
+            properties[name] = value
+          end
+
+          item[:properties] = properties
+          items << item
+        end
+        items
+      end
+    end
+
+    # Count all tag types on the page
+    # @return [Hash] Counts of different HTML elements
+    def tag_count
+      tags = {}
+      @page.css('*').each do |element|
+        tag_name = element.name.downcase
+        tags[tag_name] ||= 0
+        tags[tag_name] += 1
+      end
+      tags
+    end
+
+    # Extract RSS/Atom feeds from the page
+    # @return [Array<String>] Array of feed URLs
+    def feeds
+      @feeds ||= begin
+        feeds = []
+
+        # Look for feed link tags
+        @page.css('link[type="application/rss+xml"], link[type="application/atom+xml"]').each do |link|
+          href = link[:href]
+          feeds << make_absolute_url(href) if href
+        end
+
+        # Look for common feed patterns in links
+        links.each do |link|
+          feeds << link if link =~ %r{/(feed|rss|atom)(/|\.xml|$)}i
+        end
+
+        feeds.uniq.compact
+      end
+    end
+
+    # Extract social media profile links
+    # @return [Hash] Hash of social platform => URL
+    def social_links
+      @social_links ||= begin
+        socials = {}
+        platforms = {
+          facebook: /facebook\.com/,
+          twitter: /(twitter\.com|x\.com)/,
+          linkedin: /linkedin\.com/,
+          instagram: /instagram\.com/,
+          youtube: /youtube\.com/,
+          github: /github\.com/,
+          tiktok: /tiktok\.com/
+        }
+
+        # Check links
+        links.each do |link|
+          platforms.each do |platform, pattern|
+            socials[platform] ||= link if link.match?(pattern)
+          end
+        end
+
+        socials
+      end
+    end
+
+    # Get robots.txt URL
+    # @return [String] robots.txt URL
+    def robots_txt_url
+      "#{@url.split('/')[0..2].join('/')}/robots.txt" if @url
+    end
+
+    # Get sitemap URL
+    # @return [Array<String>] Array of sitemap URLs
+    def sitemap_url
+      @sitemap_url ||= begin
+        sitemaps = []
+
+        # Check for sitemap link tag
+        @page.css('link[rel="sitemap"]').each do |link|
+          href = link[:href]
+          sitemaps << make_absolute_url(href) if href
+        end
+
+        # Add default sitemap.xml
+        sitemaps << "#{@url.split('/')[0..2].join('/')}/sitemap.xml" if @url
+
+        sitemaps.uniq.compact
+      end
+    end
+
+    # Detect CMS and get detailed information
+    # @return [Hash] CMS information
+    def cms_info
+      @cms_info ||= begin
+        info = { name: nil, version: nil, themes: [], plugins: [] }
+
+        # WordPress detection
+        if @page.to_html.include?('wp-content') || @meta['generator']&.include?('WordPress')
+          info[:name] = 'WordPress'
+          # Try to extract version from generator meta tag
+          info[:version] = Regexp.last_match(1) if @meta['generator'] =~ /WordPress\s+([\d.]+)/
+
+          # Detect themes
+          @page.css('link[href*="wp-content/themes"]').each do |link|
+            info[:themes] << Regexp.last_match(1) if link[:href] =~ %r{themes/([^/]+)}
+          end
+
+          # Detect plugins
+          @page.css('link[href*="wp-content/plugins"], script[src*="wp-content/plugins"]').each do |elem|
+            src = elem[:href] || elem[:src]
+            info[:plugins] << Regexp.last_match(1) if src =~ %r{plugins/([^/]+)}
+          end
+        # Drupal detection
+        elsif @page.to_html.include?('Drupal') || @meta['generator']&.include?('Drupal')
+          info[:name] = 'Drupal'
+          info[:version] = Regexp.last_match(1) if @meta['generator'] =~ /Drupal\s+([\d.]+)/
+        # Joomla detection
+        elsif @meta['generator']&.include?('Joomla')
+          info[:name] = 'Joomla'
+          info[:version] = Regexp.last_match(1) if @meta['generator'] =~ /Joomla!\s+([\d.]+)/
+        # Shopify detection
+        elsif @page.to_html.include?('cdn.shopify.com') || @page.to_html.include?('Shopify')
+          info[:name] = 'Shopify'
+        # Wix detection
+        elsif @page.to_html.include?('wix.com') || @page.to_html.include?('_wix')
+          info[:name] = 'Wix'
+        # Squarespace detection
+        elsif @page.to_html.include?('squarespace')
+          info[:name] = 'Squarespace'
+        end
+
+        info[:themes].uniq!
+        info[:plugins].uniq!
+        info
+      end
+    end
+
+    # Calculate a basic accessibility score
+    # @return [Hash] Accessibility score and details
+    def accessibility_score
+      @accessibility_score ||= begin
+        score = 100
+        details = []
+
+        # Check images for alt text
+        images_without_alt = @page.css('img:not([alt])').count
+        total_images = @page.css('img').count
+
+        if total_images.positive?
+          alt_percentage = ((total_images - images_without_alt).to_f / total_images * 100).round
+          if alt_percentage < 100
+            penalty = (100 - alt_percentage) / 4 # Max 25 points penalty
+            score -= penalty
+            details << "#{images_without_alt} images missing alt text"
+          end
+        end
+
+        # Check heading hierarchy
+        h1_count = @page.css('h1').count
+        if h1_count.zero?
+          score -= 15
+          details << 'No H1 heading found'
+        elsif h1_count > 1
+          score -= 10
+          details << 'Multiple H1 headings found'
+        end
+
+        # Check for ARIA labels on interactive elements
+        buttons_without_aria = @page.css('button:not([aria-label]):not([aria-labelledby])').select do |btn|
+          btn.text.strip.empty?
+        end.count
+
+        if buttons_without_aria.positive?
+          score -= [buttons_without_aria * 5, 20].min
+          details << "#{buttons_without_aria} buttons without accessible labels"
+        end
+
+        # Check for language attribute
+        html_tag = @page.at('html')
+        if html_tag.nil? || html_tag['lang'].nil? || html_tag['lang'].empty?
+          score -= 10
+          details << 'No language attribute on HTML element'
+        end
+
+        # Check for form labels
+        inputs = @page.css('input[type="text"], input[type="email"], input[type="password"], textarea')
+        inputs_without_labels = inputs.select do |input|
+          id = input['id']
+          !id || @page.css("label[for=\"#{id}\"]").empty?
+        end.count
+
+        if inputs_without_labels.positive?
+          score -= [inputs_without_labels * 5, 15].min
+          details << "#{inputs_without_labels} form inputs without labels"
+        end
+
+        { score: [score, 0].max, details: details }
+      end
+    end
+
+    # Check if the page is mobile-friendly
+    # @return [Boolean] true if mobile-friendly
+    def mobile_friendly?
+      @mobile_friendly ||= begin
+        # Check for viewport meta tag
+        viewport = @meta['viewport']
+        has_viewport = !viewport.nil? && viewport.include?('width=device-width')
+
+        # Check for responsive CSS (media queries)
+        has_media_queries = stylesheets.any? || @page.to_html.include?('@media')
+
+        has_viewport && has_media_queries
       end
     end
 
@@ -152,7 +433,7 @@ module WebInspector
     # @return [Array<Hash>] Count results
     def counter(text, words)
       words.map do |word|
-        { word => text.scan(/#{word.downcase}/).size }
+        { word => text.scan(/#{Regexp.escape(word.downcase)}/).size }
       end
     end
 
@@ -179,6 +460,30 @@ module WebInspector
       end
     end
 
+    # Filter a list of URLs by a given domain.
+    # @param collection [Array<String>] The list of URLs to filter.
+    # @param user_domain [String] The domain to filter by.
+    # @return [Array<String>] The filtered list of URLs.
+    def filter_by_domain(collection, user_domain)
+      return [] if collection.empty?
+
+      # Handle nil user_domain
+      user_domain = @host.to_s if user_domain.nil? || user_domain.empty?
+
+      # Normalize domain for comparison
+      normalized_domain = user_domain.to_s.downcase.gsub(/\s+/, '').sub(/^www\./, '')
+
+      collection.select do |item|
+        uri = URI.parse(item.to_s)
+        next false unless uri.host
+
+        uri_host = uri.host.to_s.downcase.sub(/^www\./, '')
+        uri_host.include?(normalized_domain)
+      rescue URI::InvalidURIError, NoMethodError
+        false
+      end
+    end
+
     # Make a URL absolute
     # @param url [String] URL to make absolute
     # @return [String, nil] Absolute URL or nil if invalid
@@ -191,39 +496,31 @@ module WebInspector
       # Get base URL from the page if not already set
       if @base_url.nil?
         base_tag = @page.at_css('base[href]')
-        @base_url = base_tag ? base_tag['href'] : nil
+        @base_url = base_tag ? base_tag['href'] : ''
       end
 
       begin
         # Try joining with base URL first if available
-        if @base_url && !@base_url.empty?
-          begin
-            return URI.join(@base_url, url).to_s
-          rescue URI::InvalidURIError, URI::BadURIError
-            # Fall through to next method
-          end
-        end
+        return URI.join(@base_url, url).to_s unless @base_url.empty?
+      rescue URI::InvalidURIError, URI::BadURIError
+        # Fall through to next method
+      end
 
+      begin
         # If we have @url, try to use it
-        if @url
-          begin
-            return URI.join(@url, url).to_s
-          rescue URI::InvalidURIError, URI::BadURIError
-            # Fall through to next method
-          end
-        end
-
-        # Otherwise use a default http:// base if url is absolute path
-        return "http://#{@host}#{url}" if url.start_with?('/')
-
-        # For truly relative URLs with no base, we need to make our best guess
-        return "http://#{@host}/#{url}" if @host
-
-        # Last resort, return the original
-        url
+        return URI.join(@url, url).to_s if @url
       rescue URI::InvalidURIError, URI::BadURIError
-        url # Return original instead of nil to be more lenient
+        # Fall through to next method
       end
+
+      # For relative URLs, we need to make our best guess
+      return "http://#{@host}#{url}" if url.start_with?('/')
+      return "http://#{@host}/#{url}" if @host
+
+      # Last resort, return the original
+      url
+    rescue URI::InvalidURIError, URI::BadURIError
+      url # Return original instead of nil to be more lenient
     end
 
     # Extract a snippet from the first long paragraph

data/lib/web_inspector/meta.rb

@@ -26,6 +26,19 @@ module WebInspector
         .merge(meta_tag['property'])
         .merge(meta_tag['itemprop'] || {})
         .merge('charset' => meta_tag['charset'])
+        .merge('author' => author, 'publisher' => publisher)
+    end
+
+    def author
+      meta_tag['name']['author'] || meta_tag['property']['article:author']
+    rescue StandardError
+      nil
+    end
+
+    def publisher
+      meta_tag['property']['article:publisher'] || meta_tag['property']['og:site_name']
+    rescue StandardError
+      nil
     end
 
     def charset

data/lib/web_inspector/page.rb

@@ -19,8 +19,7 @@ require File.expand_path(File.join(File.dirname(__FILE__), 'request'))
 
 module WebInspector
   class Page
-    attr_reader :url, :scheme, :host, :port, :title, :description, :body, :meta, :links,
-                :domain_links, :domain_images, :images, :response, :status_code, :favicon
+    attr_reader :status_code
 
     DEFAULT_TIMEOUT = 30
     DEFAULT_RETRIES = 3
@@ -70,7 +69,8 @@ module WebInspector
     end
 
     # Delegate methods to inspector
-    %i[title description body links images meta].each do |method|
+    %i[title description body links images meta javascripts stylesheets language structured_data microdata
+       tag_count feeds social_links robots_txt_url sitemap_url cms_info accessibility_score mobile_friendly?].each do |method|
       define_method(method) do
         return nil unless success?
 
@@ -132,8 +132,99 @@ module WebInspector
       @inspector.domain_images(u, host)
     end
 
-    # Get full JSON representation of the page
-    #
+    # Get information about the page's security
+    # @return [Hash] Security information
+    def security_info
+      return @security_info if defined?(@security_info)
+
+      @security_info = {
+        secure: scheme == 'https',
+        hsts: response&.headers && response.headers['strict-transport-security'] ? true : false,
+        content_security_policy: response&.headers && response.headers['content-security-policy'] ? true : false
+      }
+
+      # Extract SSL/TLS info if available and using HTTPS
+      if scheme == 'https' && response&.env&.response_headers
+        @security_info[:ssl_version] = response.env[:ssl_version]
+        @security_info[:cipher_suite] = response.env[:cipher_suite]
+      end
+
+      @security_info
+    end
+
+    # Get the content type of the page
+    # @return [String, nil] Content type
+    def content_type
+      response&.headers && response.headers['content-type']
+    end
+
+    # Get the size of the page in bytes
+    # @return [Integer, nil] Size in bytes
+    def size
+      return @size if defined?(@size)
+
+      @size = if response&.headers && response.headers['content-length']
+                response.headers['content-length'].to_i
+              elsif response&.body
+                response.body.bytesize
+              end
+    end
+
+    # Get the load time of the page in seconds
+    # @return [Float, nil] Load time in seconds
+    attr_reader :load_time
+
+    # Get all JSON-LD structured data as a hash
+    # @return [Array<Hash>] Structured data
+    def json_ld
+      structured_data
+    end
+
+    # Get a hash of all technologies detected on the page
+    # @return [Hash] Detected technologies
+    def technologies
+      techs = {}
+      js_files = javascripts || []
+      css_files = stylesheets || []
+      page_body = body || ''
+      page_meta = meta || {}
+      response_headers = response&.headers || {}
+
+      # Frameworks and Libraries
+      techs[:jquery] = true if js_files.any? { |js| js.include?('jquery') } || page_body.include?('jQuery')
+      techs[:react] = true if page_body.include?('data-reactroot') || js_files.any? { |js| js.include?('react') }
+      techs[:vue] = true if page_body.include?('data-v-app') || js_files.any? { |js| js.include?('vue') }
+      techs[:angular] = true if page_body.include?('ng-version') || js_files.any? { |js| js.include?('angular') }
+      techs[:bootstrap] = true if css_files.any? do |css|
+        css.include?('bootstrap')
+      end || page_body.include?('class="container"')
+      if response_headers['x-powered-by']&.include?('Rails') || response_headers.key?('x-rails-env')
+        techs[:rails] =
+          true
+      end
+      techs[:php] = true if response_headers['x-powered-by']&.include?('PHP')
+
+      # CMS
+      techs[:wordpress] = true if page_meta['generator']&.include?('WordPress') || page_body.include?('/wp-content/')
+      techs[:shopify] = true if page_body.include?('Shopify.shop')
+
+      # Analytics
+      techs[:google_analytics] = true if js_files.any? { |js| js.include?('google-analytics.com') }
+
+      # Server
+      server = response_headers['server']
+      if server
+        techs[:server] = server
+        techs[:nginx] = true if server.include?('nginx')
+        techs[:apache] = true if server.include?('Apache')
+        techs[:iis] = true if server.include?('IIS')
+        techs[:express] = true if response_headers['x-powered-by']&.include?('Express')
+      end
+
+      techs
+    end
+
+    # Get full JSON representation of the page with all new data
     # @return [Hash] JSON representation of the page
     def to_hash
       {
@@ -146,7 +237,25 @@ module WebInspector
         'meta' => meta,
         'links' => links,
         'images' => images,
+        'javascripts' => javascripts,
+        'stylesheets' => stylesheets,
         'favicon' => favicon,
+        'language' => language,
+        'structured_data' => structured_data,
+        'microdata' => microdata,
+        'security_info' => security_info,
+        'content_type' => content_type,
+        'size' => size,
+        'load_time' => load_time,
+        'technologies' => technologies,
+        'tag_count' => tag_count,
+        'feeds' => feeds,
+        'social_links' => social_links,
+        'robots_txt_url' => robots_txt_url,
+        'sitemap_url' => sitemap_url,
+        'cms_info' => cms_info,
+        'accessibility_score' => accessibility_score,
+        'mobile_friendly' => mobile_friendly?,
         'response' => {
           'status' => status_code,
           'headers' => response&.headers || {},
@@ -166,6 +275,8 @@ module WebInspector
     private
 
     def fetch
+      start_time = Time.now
+
       session = Faraday.new(url: url) do |faraday|
         # Configure retries based on available middleware
         faraday.request :retry, { max: @retries } if defined?(Faraday::Retry)
@@ -194,6 +305,7 @@ module WebInspector
         end
 
         @url = response.env.url.to_s
+        @load_time = Time.now - start_time
         response
       rescue Faraday::TimeoutError, Faraday::ConnectionFailed => e
         retries += 1

data/lib/web_inspector/request.rb

@@ -28,6 +28,22 @@ module WebInspector
       URI(normalized_uri).port
     end
 
+    def valid?
+      !uri.nil? && !uri.host.nil?
+    rescue StandardError
+      false
+    end
+
+    def ssl?
+      scheme == 'https'
+    end
+
+    def error_message
+      return nil if valid?
+
+      'Invalid URL: Unable to parse the provided URL'
+    end
+
     private
 
     def suffix_domain
@@ -47,7 +63,11 @@ module WebInspector
     end
 
     def normalized_uri
+      return '' if uri.nil?
+
       uri.normalize.to_s
+    rescue StandardError
+      @url
     end
   end
 end

data/lib/web_inspector/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module WebInspector
-  VERSION = '1.0.0'
+  VERSION = '1.2.0'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: webinspector
 version: !ruby/object:Gem::Version
-  version: 1.0.0
+  version: 1.2.0
 platform: ruby
 authors:
 - Davide Santangelo
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2025-03-18 00:00:00.000000000 Z
+date: 2025-11-28 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake