webinspector 0.5.0 → 1.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: 943a718d012d5b472e7ecdb96d1eb4e6e178bc4a
-  data.tar.gz: c600d322e258a2efd527bde502875655f2819f86
+SHA256:
+  metadata.gz: 0413d3ff948ab6efff6a1cbe8a7844287149ad06f09353655e6cb208968f9481
+  data.tar.gz: 152b950595afb57adc522da24c6959f71d160ba903b3d01ce6ee5f6a8b4d81d2
 SHA512:
-  metadata.gz: 253f5907d503c96fc19485c58ddd59ec0373a9d4c755ffd5189bfd02bf0851692ffda30c25a7e44c7b2263f6dc300049bdd59b56fcc4f4970aca22d7d6ecb700
-  data.tar.gz: 630e4c3c60de4ef1d057180cea7e09d030de573a4d9dc248d0a61657a7f44049f54b60c633d0f7ebfbe54ee70863f8d615095a4335376d2d6a8ed86527d9ff49
+  metadata.gz: c6230493b59a0d23585be729ec98706cfdbd6852e2de2d65db83d1638f85110369d41f2275b8e0aa09b58008d53924d036840ce63523d9004f19275999be90f8
+  data.tar.gz: dad518b0b04c1e341c14c29438ebcf84f4602bf394254a4c61382ad79ce53dae2ac92f138f331d5bf089d2220011f14b0cde0a79608a91177b2bda2ab1773a96

data/Gemfile

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 source 'https://rubygems.org'
 
 # Specify your gem's dependencies in webinspector.gemspec

data/README.md

@@ -1,10 +1,10 @@
-# Webinspector
+# WebInspector
 
-Ruby gem to inspect completely a web page. It scrapes a given URL, and returns you its title, description, meta, links, images and more.
+Ruby gem to inspect web pages. It scrapes a given URL and returns its title, description, meta tags, links, images, and more.
+
+<a href="https://codeclimate.com/github/davidesantangelo/webinspector"><img src="https://codeclimate.com/github/davidesantangelo/webinspector/badges/gpa.svg" /></a>
 
-## See it in action!
 
-You can try WebInspector live at this little demo: [https://scrappet.herokuapp.com](https://scrappet.herokuapp.com)
 ## Installation
 
 Add this line to your application's Gemfile:
@@ -23,50 +23,74 @@ Or install it yourself as:
 
 ## Usage
 
-Initialize a WebInspector instance for an URL, like this:
+### Initialize a WebInspector instance
 
 ```ruby
-page = WebInspector.new('http://davidesantangelo.com')
+page = WebInspector.new('http://example.com')
 ```
 
-## Accessing response status and headers
+### With options
+
+```ruby
+page = WebInspector.new('http://example.com', {
+  timeout: 30,                         # Request timeout in seconds (default: 30)
+  retries: 3,                          # Number of retries (default: 3) 
+  headers: {'User-Agent': 'Custom UA'} # Custom HTTP headers
+})
+```
 
-You can check the status and headers from the response like this:
+### Accessing response status and headers
 
 ```ruby
 page.response.status  # 200
-page.response.headers # { "server"=>"apache", "content-type"=>"text/html; charset=utf-8", "cache-control"=>"must-revalidate, private, max-age=0", ... }
+page.response.headers # { "server"=>"apache", "content-type"=>"text/html; charset=utf-8", ... }
+page.status_code      # 200
+page.success?         # true if the page was loaded successfully
+page.error_message    # returns the error message if any
 ```
 
-## Accessing inpsected data
-
-You can see the data like this:
+### Accessing page data
 
 ```ruby
-page.url                 # URL of the page
-page.scheme              # Scheme of the page (http, https)
-page.host                # Hostname of the page (like, davidesantangelo.com, without the scheme)
-page.port                # Port of the page
-page.title               # title of the page from the head section, as string
-page.description         # description of the page
-page.links               # every link found
-page.images              # every image found
-page.meta                # metatags of the page
+page.url           # URL of the page
+page.scheme        # Scheme of the page (http, https)
+page.host          # Hostname of the page (like, example.com, without the scheme)
+page.port          # Port of the page
+page.title         # title of the page from the head section
+page.description   # description of the page
+page.links         # array of all links found on the page (absolute URLs)
+page.images        # array of all images found on the page (absolute URLs)
+page.meta          # meta tags of the page
+page.favicon       # favicon URL if available
 ```
 
-## Accessing meta tags
+### Working with meta tags
 
 ```ruby
-page.meta                 # metatags of the page
+page.meta                 # all meta tags
 page.meta['description']  # meta description
 page.meta['keywords']     # meta keywords
+page.meta['og:title']     # OpenGraph title
+```
+
+### Filtering links and images by domain
+
+```ruby
+page.domain_links('example.com')  # returns only links pointing to example.com
+page.domain_images('example.com') # returns only images hosted on example.com
 ```
 
-## Find words (as array)
+### Searching for words
+
 ```ruby
-page.find(["word1, word2"]) # return {"word1"=>3, "word2"=>1}
+page.find(["ruby", "rails"]) # returns [{"ruby"=>3}, {"rails"=>1}]
 ```
 
+### Export all data to JSON
+
+```ruby
+page.to_hash # returns a hash with all page data
+```
 
 ## Contributors
 
@@ -74,13 +98,13 @@ page.find(["word1, word2"]) # return {"word1"=>3, "word2"=>1}
   * Sam Nissen ([@samnissen](https://github.com/samnissen))
 
 ## License
-The webinspector GEM is released under the MIT License.
+
+The WebInspector gem is released under the MIT License.
 
 ## Contributing
 
-1. Fork it ( https://github.com/[my-github-username]/webinspector/fork )
+1. Fork it ( https://github.com/davidesantangelo/webinspector/fork )
 2. Create your feature branch (`git checkout -b my-new-feature`)
 3. Commit your changes (`git commit -am 'Add some feature'`)
 4. Push to the branch (`git push origin my-new-feature`)
 5. Create a new Pull Request
->>>>>>> develop

data/Rakefile

@@ -1,2 +1,3 @@
-require "bundler/gem_tasks"
+# frozen_string_literal: true
 
+require 'bundler/gem_tasks'

data/bin/console

@@ -1,7 +1,8 @@
 #!/usr/bin/env ruby
+# frozen_string_literal: true
 
-require "bundler/setup"
-require "webinspector"
+require 'bundler/setup'
+require 'webinspector'
 
 # You can add fixtures and/or initialization code here to make experimenting
 # with your gem easier. You can also use a different console, if you like.
@@ -10,5 +11,5 @@ require "webinspector"
 # require "pry"
 # Pry.start
 
-require "irb"
+require 'irb'
 IRB.start

data/lib/web_inspector/inspector.rb

@@ -1,144 +1,236 @@
+# frozen_string_literal: true
+
 require File.expand_path(File.join(File.dirname(__FILE__), 'meta'))
 
 module WebInspector
   class Inspector
+    attr_reader :page, :url, :host, :meta
 
     def initialize(page)
       @page = page
       @meta = WebInspector::Meta.new(page).meta
+      @base_url = nil
+    end
+
+    def set_url(url, host)
+      @url = url
+      @host = host
     end
 
     def title
-      @page.css('title').inner_text.strip rescue nil
+      @page.css('title').inner_text.strip
+    rescue StandardError
+      nil
     end
 
     def description
-      @meta['description'] || snippet
+      @meta['description'] || @meta['og:description'] || snippet
     end
 
     def body
       @page.css('body').to_html
     end
 
-    def meta
-      @meta
-    end
-
+    # Search for specific words in the page content
+    # @param words [Array<String>] List of words to search for
+    # @return [Array<Hash>] Counts of word occurrences
     def find(words)
-      text = @page.at('html').inner_text   
+      text = @page.at('html').inner_text
       counter(text.downcase, words)
     end
 
+    # Get all links from the page
+    # @return [Array<String>] Array of URLs
     def links
-      get_new_links unless @links
-      return @links
-    end
-    
-    def domain_links(user_domain, host)
-      @host ||= host
-      
-      validated_domain_uri = validate_url_domain("http://#{user_domain.downcase.gsub(/\s+/, '')}")
-      raise "Invalid domain provided" unless validated_domain_uri
-      
-      domain = validated_domain_uri.domain
-      
-      domain_links = []
-      
-      links.each do |l|
-        
-        u = validate_url_domain(l)
-        next unless u && u.domain
-        
-        domain_links.push(l) if domain == u.domain.downcase
+      @links ||= begin
+        links = []
+        @page.css('a').each do |a|
+          href = a[:href]
+          next unless href
+
+          # Skip javascript and mailto links
+          next if href.start_with?('javascript:', 'mailto:', 'tel:')
+
+          # Clean and normalize URL
+          href = href.strip
+
+          begin
+            absolute_url = make_absolute_url(href)
+            links << absolute_url if absolute_url
+          rescue URI::InvalidURIError
+            # Skip invalid URLs
+          end
+        end
+        links.uniq
       end
-      
-      return domain_links.compact
     end
-    
-    def domain_images(user_domain, host)
+
+    # Get links from a specific domain
+    # @param user_domain [String] Domain to filter links by
+    # @param host [String] Current host
+    # @return [Array<String>] Filtered links
+    def domain_links(user_domain, host = nil)
       @host ||= host
-      
-      validated_domain_uri = validate_url_domain("http://#{user_domain.downcase.gsub(/\s+/, '')}")
-      raise "Invalid domain provided" unless validated_domain_uri
-      
-      domain = validated_domain_uri.domain
-      
-      domain_images = []
-      
-      images.each do |img|
-        u = validate_url_domain(img)
-        next unless u && u.domain
-        
-        domain_images.push(img) if u.domain.downcase.end_with?(domain)
+
+      return [] if links.empty?
+
+      # Handle nil user_domain
+      user_domain = @host.to_s if user_domain.nil? || user_domain.empty?
+
+      # Normalize domain for comparison
+      user_domain = user_domain.to_s.downcase.gsub(/\s+/, '')
+      user_domain = user_domain.sub(/^www\./, '') # Remove www prefix for comparison
+
+      links.select do |link|
+        uri = URI.parse(link.to_s)
+        next false unless uri.host # Skip URLs without hosts
+
+        uri_host = uri.host.to_s.downcase
+        uri_host = uri_host.sub(/^www\./, '') # Remove www prefix for comparison
+        uri_host.include?(user_domain)
+      rescue URI::InvalidURIError, NoMethodError
+        false
       end
-      
-      return domain_images.compact
     end
-    
-    # Normalize and validate the URLs on the page for comparison
-    def validate_url_domain(u)
-      # Enforce a few bare standards before proceeding
-      u = "#{u}" 
-      u = "/" if u.empty?
-      
-      begin
-        # Look for evidence of a host. If this is a relative link
-        # like '/contact', add the page host.
-        domained_url   = @host + u unless (u.split("/").first || "").match(/(\:|\.)/)
-        domained_url ||= u
-        
-        # http the URL if it is missing
-        httpped_url   = "http://" + domained_url unless domained_url[0..3] == 'http'
-        httpped_url ||= domained_url
-        
-        # Make sure the URL parses
-        uri     = URI.parse(httpped_url)
-        
-        # Make sure the URL passes ICANN rules.
-        # The PublicSuffix object splits the domain and subdomain
-        # (unlike URI), which allows more liberal URL matching.
-        return PublicSuffix.parse(uri.host)
-      rescue URI::InvalidURIError, PublicSuffix::DomainInvalid => e
-        return false
+
+    # Get all images from the page
+    # @return [Array<String>] Array of image URLs
+    def images
+      @images ||= begin
+        images = []
+        @page.css('img').each do |img|
+          src = img[:src]
+          next unless src
+
+          # Clean and normalize URL
+          src = src.strip
+
+          begin
+            absolute_url = make_absolute_url(src)
+            images << absolute_url if absolute_url
+          rescue URI::InvalidURIError, URI::BadURIError
+            # Skip invalid URLs
+          end
+        end
+        images.uniq.compact
       end
     end
 
-    def images
-      get_new_images unless @images
-      return @images
+    # Get images from a specific domain
+    # @param user_domain [String] Domain to filter images by
+    # @param host [String] Current host
+    # @return [Array<String>] Filtered images
+    def domain_images(user_domain, host = nil)
+      @host ||= host
+
+      return [] if images.empty?
+
+      # Handle nil user_domain
+      user_domain = @host.to_s if user_domain.nil? || user_domain.empty?
+
+      # Normalize domain for comparison
+      user_domain = user_domain.to_s.downcase.gsub(/\s+/, '')
+      user_domain = user_domain.sub(/^www\./, '') # Remove www prefix for comparison
+
+      images.select do |img|
+        uri = URI.parse(img.to_s)
+        next false unless uri.host # Skip URLs without hosts
+
+        uri_host = uri.host.to_s.downcase
+        uri_host = uri_host.sub(/^www\./, '') # Remove www prefix for comparison
+        uri_host.include?(user_domain)
+      rescue URI::InvalidURIError, NoMethodError
+        false
+      end
     end
 
     private
-    
-    def counter(text, words)
-      results = []
-      hash = Hash.new
 
-      words.each do |word|
-        hash[word] = text.scan(/#{word.downcase}/).size
-        results.push(hash)
-        hash = Hash.new
+    # Count occurrences of words in text
+    # @param text [String] Text to search in
+    # @param words [Array<String>] Words to find
+    # @return [Array<Hash>] Count results
+    def counter(text, words)
+      words.map do |word|
+        { word => text.scan(/#{word.downcase}/).size }
       end
-      return results
     end
 
-    def get_new_images
-      @images = []
-      @page.css("img").each do |img|
-        @images.push((img[:src].to_s.start_with? @url.to_s) ? img[:src] : URI.join(url, img[:src]).to_s) if (img and img[:src])
+    # Validate a URL domain
+    # @param u [String] URL to validate
+    # @return [PublicSuffix::Domain, false] Domain object or false if invalid
+    def validate_url_domain(u)
+      u = u.to_s
+      u = '/' if u.empty?
+
+      begin
+        domained_url = if !(u.split('/').first || '').match(/(:|\.)/)
+                         @host + u
+                       else
+                         u
+                       end
+
+        httpped_url = domained_url.start_with?('http') ? domained_url : "http://#{domained_url}"
+        uri = URI.parse(httpped_url)
+
+        PublicSuffix.parse(uri.host)
+      rescue URI::InvalidURIError, PublicSuffix::DomainInvalid
+        false
       end
     end
-    
-    def get_new_links
-      @links = []
-      @page.css("a").each do |a|
-        @links.push((a[:href].to_s.start_with? @url.to_s) ? a[:href] : URI.join(@url, a[:href]).to_s) if (a and a[:href])
+
+    # Make a URL absolute
+    # @param url [String] URL to make absolute
+    # @return [String, nil] Absolute URL or nil if invalid
+    def make_absolute_url(url)
+      return nil if url.nil? || url.empty?
+
+      # If it's already absolute, return it
+      return url if url.start_with?('http://', 'https://')
+
+      # Get base URL from the page if not already set
+      if @base_url.nil?
+        base_tag = @page.at_css('base[href]')
+        @base_url = base_tag ? base_tag['href'] : nil
+      end
+
+      begin
+        # Try joining with base URL first if available
+        if @base_url && !@base_url.empty?
+          begin
+            return URI.join(@base_url, url).to_s
+          rescue URI::InvalidURIError, URI::BadURIError
+            # Fall through to next method
+          end
+        end
+
+        # If we have @url, try to use it
+        if @url
+          begin
+            return URI.join(@url, url).to_s
+          rescue URI::InvalidURIError, URI::BadURIError
+            # Fall through to next method
+          end
+        end
+
+        # Otherwise use a default http:// base if url is absolute path
+        return "http://#{@host}#{url}" if url.start_with?('/')
+
+        # For truly relative URLs with no base, we need to make our best guess
+        return "http://#{@host}/#{url}" if @host
+
+        # Last resort, return the original
+        url
+      rescue URI::InvalidURIError, URI::BadURIError
+        url # Return original instead of nil to be more lenient
       end
     end
 
+    # Extract a snippet from the first long paragraph
+    # @return [String] Text snippet
     def snippet
       first_long_paragraph = @page.search('//p[string-length() >= 120]').first
-      first_long_paragraph ? first_long_paragraph.text : ''
+      first_long_paragraph ? first_long_paragraph.text.strip[0..255] : ''
     end
   end
-end
+end

data/lib/web_inspector/meta.rb

@@ -1,15 +1,18 @@
+# frozen_string_literal: true
+
 module WebInspector
   class Meta
-  	def initialize(page)
+    def initialize(page)
       @page = page
     end
 
     def meta_tags
       {
-        'name'        => meta_tags_by('name'),
-        'http-equiv'  => meta_tags_by('http-equiv'),
-        'property'    => meta_tags_by('property'),
-        'charset'     => [charset_from_meta_charset]
+        'name' => meta_tags_by('name'),
+        'http-equiv' => meta_tags_by('http-equiv'),
+        'property' => meta_tags_by('property'),
+        'charset' => [charset_from_meta_charset],
+        'itemprop' => meta_tags_by('itemprop') # Add support for schema.org microdata
       }
     end
 
@@ -19,30 +22,48 @@ module WebInspector
 
     def meta
       meta_tag['name']
-          .merge(meta_tag['http-equiv'])
-          .merge(meta_tag['property'])
-          .merge('charset' => meta_tag['charset'])
+        .merge(meta_tag['http-equiv'])
+        .merge(meta_tag['property'])
+        .merge(meta_tag['itemprop'] || {})
+        .merge('charset' => meta_tag['charset'])
     end
 
     def charset
-      @charset ||= (charset_from_meta_charset || charset_from_meta_content_type)
+      @charset ||= charset_from_meta_charset || charset_from_meta_content_type || charset_from_header || 'utf-8'
     end
 
     private
 
     def charset_from_meta_charset
-      @page.css('meta[charset]')[0].attributes['charset'].value rescue nil
+      @page.css('meta[charset]')[0].attributes['charset'].value
+    rescue StandardError
+      nil
     end
 
     def charset_from_meta_content_type
-      @page.css("meta[http-equiv='Content-Type']")[0].attributes['content'].value.split(';')[1].split('=')[1] rescue nil
+      @page.css("meta[http-equiv='Content-Type']")[0].attributes['content'].value.split(';')[1].strip.split('=')[1]
+    rescue StandardError
+      nil
+    end
+
+    def charset_from_header
+      # Try to get charset from Content-Type header if available
+      nil
     end
 
-   	def meta_tags_by(attribute)
+    def meta_tags_by(attribute)
       hash = {}
       @page.css("meta[@#{attribute}]").map do |tag|
-        name    = tag.attributes[attribute].value.downcase rescue nil
-        content = tag.attributes['content'].value rescue nil
+        name = begin
+          tag.attributes[attribute].value.downcase
+        rescue StandardError
+          nil
+        end
+        content = begin
+          tag.attributes['content'].value
+        rescue StandardError
+          nil
+        end
 
         if name && content
           hash[name] ||= []
@@ -64,4 +85,4 @@ module WebInspector
       end
     end
   end
-end
+end