wgit 0.0.9 → 0.0.10

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: ae86258e3aac086f2215d1fb3e3b871cd4f4839884eb7c359ac9863148f1a307
-  data.tar.gz: 2eafa3a2b7b6d6ff99aaf00466ccd5f9214f049a9f5836b45aaf6d17bfbe226b
+  metadata.gz: c4dc572e7d48a95d423e175ad2d6a791be52bf56f6c391e9152c075f45672ee8
+  data.tar.gz: fd6a9c9d1e38906f500543ae92169f7bcb9e64de1567f4f29ec24f7ca74c60d8
 SHA512:
-  metadata.gz: 5735051c62d3d22db75a42c7d33cd8b1f78d4500b27c0e136980382ffb0e6830ee8d355a0f9151c4d844dd0eb91dc860cd5bd1855ec68099985c34d55ba1a3aa
-  data.tar.gz: 0b5ab8f7f60e69f791fd4b51995f5dbf2f38a77954b164062b64b982847b7fd3844b016fbfaff186a7178f68d2fedc8736fd60b8c16ae82bc11c7a84a5892e42
+  metadata.gz: f4fc425aa1b25254dba343151794893ff26a3682e58dd08bdc918c180da89ecb08cf0f1013837e41527f564d36bb0784c6ecd204e53c1276cb5b32401c88ffab
+  data.tar.gz: 2ce1250ad7312257bc021e7414f164ec174e07e469a6956478cccaa7b05f159981a7a69ef6b713db5454233c8a03ea8487f14184c1e26dcab28ed8e81250507d

data/lib/wgit/crawler.rb

@@ -106,7 +106,8 @@ module Wgit
       doc = crawl_url(base_url, &block)
       return nil if doc.nil?
     
-      crawled_urls  = []
+      path = base_url.path.empty? ? '/' : base_url.path
+      crawled_urls  = [path]
       external_urls = doc.external_links
       internal_urls = doc.internal_links
     
@@ -149,7 +150,10 @@ module Wgit
     def fetch(url)
       response = resolve(url)
       response.body.empty? ? nil : response.body
-    rescue
+    rescue Exception => ex
+      Wgit.logger.debug(
+        "Wgit::Crawler#fetch('#{url}') exception: #{ex.message}"
+      )
       nil
     end
 
@@ -158,12 +162,16 @@ module Wgit
     # an exception. Redirects can be disabled by setting `redirect_limit: 1`.
     # The Net::HTTPResponse will be returned.
     def resolve(url, redirect_limit: 5)
-      redirect_count = 0
+      redirect_count = -1
       begin
         raise "Too many redirects" if redirect_count >= redirect_limit
-        response = Net::HTTP.get_response(URI.parse(url))
-        url = response['location']
         redirect_count += 1
+
+        response = Net::HTTP.get_response(URI(url))
+        location = Wgit::Url.new(response.fetch('location', ''))
+        if not location.empty?
+          url = location.is_relative? ? url.to_base.concat(location) : location
+        end
       end while response.is_a?(Net::HTTPRedirection)
       response
     end

data/lib/wgit/document.rb

@@ -62,6 +62,8 @@ module Wgit
         @html = html ||= ""
         @doc = init_nokogiri
         @score = 0.0
+
+        process_url_and_html
         
         # Dynamically run the init_*_from_html methods.
         Document.private_instance_methods(false).each do |method|
@@ -80,7 +82,9 @@ module Wgit
         @html = obj.fetch("html", "")
         @doc = init_nokogiri
         @score = obj.fetch("score", 0.0)
-        
+
+        process_url_and_html
+
         # Dynamically run the init_*_from_object methods.
         Document.private_instance_methods(false).each do |method|
           if method.to_s.start_with?("init_") && 
@@ -174,7 +178,7 @@ module Wgit
     # @return [Boolean] True if @html is nil/empty, false otherwise.
     def empty?
       return true if @html.nil?
-      @html.strip.empty?
+      @html.empty?
     end
     
     # Uses Nokogiri's xpath method to search the doc's html and return the 
@@ -194,47 +198,54 @@ module Wgit
     def css(selector)
       @doc.css(selector)
     end
-    
-    # Get all internal links of this Document.
+
+    # Get all internal links of this Document in relative form. Internal
+    # meaning a link to another page on this website. Also see
+    # Wgit::Document#internal_full_links.
     #
     # @return [Array<Wgit::Url>] self's internal/relative URL's.
     def internal_links
       return [] if @links.empty?
-      @links.reject do |link|
-        begin
-          not link.relative_link?
+
+      links = @links.
+        reject do |link|
+          not link.relative_link?(base: @url.to_base)
         rescue
           true
-        end
-      end
+        end.
+        map(&:to_path)
+
+      process_arr(links)
     end
     
     # Get all internal links of this Document and append them to this
-    # Document's base URL.
+    # Document's base URL making them absolute. Also see
+    # Wgit::Document#internal_links.
     #
     # @return [Array<Wgit::Url>] self's internal/relative URL's in absolute
     #   form.
     def internal_full_links
       in_links = internal_links
       return [] if in_links.empty?
-      in_links.map do |link|
-        link.replace("/" + link) unless link.start_with?("/")
-        Wgit::Url.new(@url.to_base + link)
-      end
+      in_links.map { |link| @url.to_base.concat(link) }
     end
   
-    # Get all external links of this Document.
+    # Get all external links of this Document. External meaning a link to
+    # another website.
     #
     # @return [Array<Wgit::Url>] self's external/absolute URL's.
     def external_links
       return [] if @links.empty?
-      @links.reject do |link|
-        begin
-          link.relative_link?
+
+      links = @links.
+        reject do |link|
+          link.relative_link?(base: @url.to_base)
         rescue
           true
-        end
-      end
+        end.
+        map { |link| link.end_with?('/') ? link.chop : link }
+
+      process_arr(links)
     end
   
     # Searches against the @text for the given search query.
@@ -253,8 +264,8 @@ module Wgit
     #   sentence.
     # @return [Array<String>] Representing the search results.
     def search(query, sentence_limit = 80)
-      raise "A search value must be provided" if query.empty?
-      raise "The sentence length value must be even" if sentence_limit.odd?
+      raise "A search query must be provided" if query.empty?
+      raise "The sentence_limit value must be even" if sentence_limit.odd?
     
       results = {}
       regex = Regexp.new(query, Regexp::IGNORECASE)
@@ -469,29 +480,16 @@ module Wgit
       if array.is_a?(Array)
         array.map! { |str| process_str(str) }
         array.reject! { |str| str.is_a?(String) ? str.empty? : false }
+        array.compact!
         array.uniq!
       end
       array
     end
-  
-    # Modifies internal links by removing this doc's base or host URL, if
-    # present. http://www.google.co.uk/about.html (with or without the 
-    # protocol prefix) will become about.html meaning it'll appear within 
-    # Document#internal_links.
-    def process_internal_links(links)
-      links.map! do |link|
-        host_or_base =  if link.start_with?("http")
-                          @url.base
-                        else
-                          @url.host
-                        end
-        if link.start_with?(host_or_base)
-          link.sub!(host_or_base, "")
-          link.replace(link[1..-1]) if link.start_with?("/")
-          link.strip!
-        end
-        link
-      end
+
+    # Ensure the @url and @html Strings are correctly encoded etc.
+    def process_url_and_html
+      @url = process_str(@url)
+      @html = process_str(@html)
     end
 
     ### Default init_* (Document extension) methods. ###
@@ -547,7 +545,6 @@ module Wgit
       xpath = "//a/@href"
       result = find_in_html(xpath, singleton: false) do |links|
         if links
-          links.reject! { |link| link == "/" }
           links.map! do |link|
             begin
               Wgit::Url.new(link)
@@ -555,8 +552,7 @@ module Wgit
               nil
             end
           end
-          links.reject! { |link| link.nil? }
-          process_internal_links(links)
+          links.compact!
         end
         links
       end

data/lib/wgit/url.rb

@@ -95,21 +95,29 @@ module Wgit
       url
     end
   
-    # Returns if link is a relative or absolute Url. How it works:
-    # URI.split("http://www.google.co.uk/about.html") returns the following:
-    # array[2]: "www.google.co.uk", array[5]: "/about.html".
-    # This means that all external links in a page are expected to have a 
-    # protocol prefix e.g. "http://", otherwise the link is treated as an 
-    # internal link (regardless of whether it is valid or not).
+    # Returns if link is a relative or absolute Url.
+    # All external links in a page are expected to have a protocol prefix e.g.
+    # "http://", otherwise the link is treated as an internal link (regardless
+    # of whether it is valid or not). The only exception is if base is provided
+    # and link is a page within that site; then the link is relative.
     #
     # @param link [Wgit::Url, String] The url to test if relative or not.
+    # @param base [String] The Url base e.g. http://www.google.co.uk.
     # @return [Boolean] True if relative, false if absolute.
     # @raise [RuntimeError] If the link is invalid.
-    def self.relative_link?(link)
-      link_segs = URI.split(link)
-      if not link_segs[2].nil? and not link_segs[2].empty?
-        false
-      elsif not link_segs[5].nil? and not link_segs[5].empty?
+    def self.relative_link?(link, base: nil)
+      if base and URI(base).host.nil?
+        raise "Invalid base, must contain protocol prefix: #{base}"
+      end
+      
+      uri = URI(link)
+      if not uri.host.nil? and not uri.host.empty?
+        if base
+          uri.host == URI(base).host
+        else
+          false
+        end
+      elsif not uri.path.nil? and not uri.path.empty?
         true
       else
         raise "Invalid link: #{link}"
@@ -128,11 +136,14 @@ module Wgit
       Wgit::Url.new(url + "/" + link)
     end
   
-    # Returns if self is a relative or absolute Url.
+    # Returns if self is a relative or absolute Url. If base is provided and
+    # self is a page within that site then the link is relative.
+    # See Wgit.relative_link? for more information.
+    #
     # @return [Boolean] True if relative, false if absolute.
     # @raise [RuntimeError] If the link is invalid.
-    def relative_link?
-      Wgit::Url.relative_link?(self)
+    def relative_link?(base: nil)
+      Wgit::Url.relative_link?(self, base: base)
     end
 
     # Determines if self is a valid Url or not.
@@ -142,7 +153,7 @@ module Wgit
       Wgit::Url.valid?(self)
     end
   
-    # Concats self (Url) and the link.
+    # Concats self and the link.
     #
     # @param link [Wgit::Url, String] The link to concat with self.
     # @return [Wgit::Url] self + "/" + link
@@ -172,6 +183,14 @@ module Wgit
     def to_url
       self
     end
+
+    # Returns a new Wgit::Url containing just the scheme/protocol of this URL
+    # e.g. Given http://www.google.co.uk, http is returned.
+    #
+    # @return [Wgit::Url] Containing just the scheme/protocol.
+    def to_scheme
+      Wgit::Url.new(@uri.scheme)
+    end
   
     # Returns a new Wgit::Url containing just the host of this URL e.g.
     # Given http://www.google.co.uk/about.html, www.google.co.uk is returned.
@@ -181,24 +200,54 @@ module Wgit
       Wgit::Url.new(@uri.host)
     end
   
-    # Returns the base of this URL e.g. the protocol and host combined.
-    # How it works:
-    # URI.split("http://www.google.co.uk/about.html") returns the following:
-    # array[0]: "http://", array[2]: "www.google.co.uk", which we use.
+    # Returns only the base of this URL e.g. the protocol and host combined.
     #
-    # @return [Wgit::Url] Base of self (Url) e.g. http://www.google.co.uk.
+    # @return [Wgit::Url] Base of self e.g. http://www.google.co.uk.
     def to_base
       if Wgit::Url.relative_link?(self)
         raise "A relative link doesn't have a base URL: #{self}"
       end
-      url_segs = URI.split(self)
-      if url_segs[0].nil? or url_segs[2].nil? or url_segs[2].empty?
+      if @uri.scheme.nil? or @uri.host.nil? or @uri.host.empty?
         raise "Both a protocol and host are needed: #{self}"
       end
-      base = "#{url_segs[0]}://#{url_segs[2]}"
+      base = "#{@uri.scheme}://#{@uri.host}"
       Wgit::Url.new(base)
     end
-  
+
+    # Returns the path of this URL e.g. the bit after the host without slashes.
+    # For example:
+    # Wgit::Url.new("http://www.google.co.uk/about.html/").to_path returns
+    # "about.html". See Wgit::Url#to_endpoint if you want the slashes.
+    #
+    # @return [Wgit::Url] Path of self e.g. about.html.
+    def to_path
+      path = @uri.path
+      return Wgit::Url.new(path) if path == '/'
+      path = path[1..-1] if path.start_with?('/')
+      path.chop! if path.end_with?('/')
+      Wgit::Url.new(path)
+    end
+
+    # Returns the endpoint of this URL e.g. the bit after the host with any
+    # slashes included. For example:
+    # Wgit::Url.new("http://www.google.co.uk/about.html/").to_endpoint returns
+    # "/about.html/". See Wgit::Url#to_path if you don't want the slashes.
+    #
+    # @return [Wgit::Url] Endpoint of self e.g. /about.html/.
+    def to_endpoint
+      endpoint = @uri.path
+      endpoint = '/' + endpoint unless endpoint.start_with?('/')
+      Wgit::Url.new(endpoint)
+    end
+
+    # Returns a new Wgit::Url containing just the query string of this URL
+    # e.g. Given http://google.com?q=ruby, ruby is returned.
+    #
+    # @return [Wgit::Url] Containing just the query string.
+    def to_query_string
+      Wgit::Url.new(@uri.query)
+    end
+
     # Returns a Hash containing this Url's instance vars excluding @uri.
     # Used when storing the URL in a Database e.g. MongoDB etc.
     #
@@ -210,9 +259,20 @@ module Wgit
     end
   
     alias :to_hash :to_h
+    alias :uri :to_uri
+    alias :url :to_url
+    alias :scheme :to_scheme
+    alias :to_protocol :to_scheme
+    alias :protocol :to_scheme
     alias :host :to_host
     alias :base :to_base
+    alias :path :to_path
+    alias :endpoint :to_endpoint
+    alias :query_string :to_query_string
+    alias :query :to_query_string
     alias :internal_link? :relative_link?
+    alias :is_relative? :relative_link?
+    alias :is_internal? :relative_link?
     alias :crawled? :crawled
   end
 end

data/lib/wgit/version.rb

@@ -3,5 +3,5 @@
 # @author Michael Telford
 module Wgit
   # The current gem version of Wgit.
-  VERSION = "0.0.9".freeze
+  VERSION = "0.0.10".freeze
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: wgit
 version: !ruby/object:Gem::Version
-  version: 0.0.9
+  version: 0.0.10
 platform: ruby
 authors:
 - Michael Telford
@@ -28,16 +28,16 @@ dependencies:
   name: yard
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '0.9'
+        version: 0.9.20
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '0.9'
+        version: 0.9.20
 - !ruby/object:Gem::Dependency
   name: byebug
   requirement: !ruby/object:Gem::Requirement
@@ -108,6 +108,34 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '1.3'
+- !ruby/object:Gem::Dependency
+  name: webmock
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.6'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.6'
+- !ruby/object:Gem::Dependency
+  name: rack
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
 - !ruby/object:Gem::Dependency
   name: nokogiri
   requirement: !ruby/object:Gem::Requirement
@@ -128,21 +156,21 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.6'
+        version: 2.8.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.6'
-description: Wgit is a WWW indexer/scraper which crawls URL's, retrieves and serialises
-  their page contents for later use. You can use Wgit to copy entire websites if required.
-  Wgit also provides a means to search indexed documents stored in a database. Therefore,
-  this library provides the main components of a WWW search engine. The Wgit API is
-  easily extended allowing you to pull out the parts of a webpage that are important
-  to you, the code snippets or images for example. As Wgit is a library, it's very
-  useful in many different application types.
+        version: 2.8.0
+description: Fundamentally, Wgit is a WWW indexer/scraper which crawls URL's, retrieves
+  and serialises their page contents for later use. You can use Wgit to copy entire
+  websites if required. Wgit also provides a means to search indexed documents stored
+  in a database. Therefore, this library provides the main components of a WWW search
+  engine. The Wgit API is easily extended allowing you to pull out the parts of a
+  webpage that are important to you, the code snippets or images for example. As Wgit
+  is a library, it has uses in many different application types.
 email: michael.telford@live.com
 executables: []
 extensions: []
@@ -166,6 +194,7 @@ licenses:
 - MIT
 metadata:
   source_code_uri: https://github.com/michaeltelford/wgit
+  yard.run: yri
   allowed_push_host: https://rubygems.org
 post_install_message: 
 rdoc_options: []
@@ -183,9 +212,9 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.7.8
+rubygems_version: 2.7.6
 signing_key: 
 specification_version: 4
-summary: Wgit is the ruby version of GNU's wget with an easy to use API for programmatic
-  web scraping, indexing and searching.
+summary: Wgit is a Ruby gem similar in nature to GNU's `wget`. It provides an easy
+  to use API for programmatic web scraping, indexing and searching.
 test_files: []