site_mapper 0.0.3 → 0.0.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 358ce69fd3d2bf6f019b176cf5d3a86c381fd4c4
-  data.tar.gz: 710c2f869c310189d9f91cac438868797dfba59b
+  metadata.gz: b8ce68b3debd6d26847da79293a2c84411423cf7
+  data.tar.gz: 471bea81d090f201590b1d56ab99e7d7de09e21c
 SHA512:
-  metadata.gz: c592d4f952f005346f45ddd99686c9882d362612b6962ead35c83962d41ef9bb0d3e64565c74d905ad7abf336c896131633ba51574b16b5816c9d419ea2bc5c7
-  data.tar.gz: 0c98a7f7d69148b9a9e41ab89e80c9fd4c737df87e48f15424234576c7cc676671098cc512953a401bbf3bf2112875e7dedcb02b5f32ad589a8b7217206bc9c0
+  metadata.gz: 158d46c3b674c7854eab08413bfb2b58656712d39bb29ca714d6c2997e8c826508542ac13e6a20286b6066b30ac51d223343314cfb77e7d072c9b51da5fa01d9
+  data.tar.gz: d7467e386d748bc41ef3f72180e36a321fa0d0d47a9e92a6e76d5aa202c9f2f38c784ff8f896359e04f5fd9f6aa55140da526e03d1e8d147e0dc73d5308c3f13

data/lib/site_mapper/crawl_url.rb

@@ -1,4 +1,5 @@
 module SiteMapper
+  # Crawl URL formatter
   class CrawlUrl
     attr_reader :resolved_base_url, :base_hostname
 
@@ -8,6 +9,14 @@ module SiteMapper
       @resolved_base_url.prepend('http://') unless @resolved_base_url.start_with?('http')
     end
 
+    # Given a link it constructs the absolute path,
+    # if valid URL & URL has same domain as @resolved_base_url.
+    # @return [String] with absolute path to resource.
+    # @param [String, String] raw_url from link element and current page URL
+    # @example Construct absolute URL for '/path', example.com
+    #   cu = CrawlUrl.new('example.com')
+    #   cu.absolute_url_from('/path', 'example.com/some/path')
+    #   # => http://example.com/some/path
     def absolute_url_from(raw_url, get_url)
       return nil unless eligible_url?(raw_url)
       parsed_url = URI.parse(raw_url) rescue URI.parse('')  

data/lib/site_mapper/crawler.rb

@@ -17,14 +17,26 @@ module SiteMapper
       @fetch_queue << @crawl_url.resolved_base_url
     end
 
+    # @see #collect_urls
     def self.collect_urls(base_url)
-      new(base_url).collect_urls
+      new(base_url).collect_urls { |url| yield(url) }
     end
 
+    # Collects all links on domain for domain
+    # @return [Array] with links.
+    # @example URLs for example.com
+    #   crawler = Crawler.new('example.com')
+    #   crawler.collect_urls
+    # @example URLs for example.com with block
+    #   crawler = Crawler.new('example.com')
+    #   crawler.collect_urls do |new_url|
+    #     puts "New URL found: #{new_url}"
+    #   end
     def collect_urls
       until @fetch_queue.empty?
         url = @fetch_queue.first
         @fetch_queue.delete(@fetch_queue.first)
+        Thread.new { yield(url) if block_given? }
         page_links(url)
       end
       puts "Crawling finished, #{@processed.length} links found"
@@ -43,8 +55,8 @@ module SiteMapper
       link_elements.each do |page_link|
         absolute_url = @crawl_url.absolute_url_from(page_link.attr('href'), get_url)
         if absolute_url
-          resolved_url = resolve(absolute_url)
-          @fetch_queue << resolved_url if !@processed.include?(resolved_url)
+          url = resolve(absolute_url)
+          @fetch_queue << url unless @processed.include?(url)
         end
       end
     end

data/lib/site_mapper/version.rb

@@ -1,3 +1,3 @@
 module SiteMapper
-  VERSION = '0.0.3'
+  VERSION = '0.0.4'
 end

data/lib/site_mapper.rb

@@ -6,7 +6,13 @@ require 'site_mapper/request'
 require 'site_mapper/crawler'
 require 'site_mapper/crawl_url'
 
+# Find all links on domain to domain
 module SiteMapper
+  # Returns all links found on domain to domain.
+  # @return [Array] with links.
+  # @param [String] URL for domain
+  # @example Collect all URLs from example.com
+  #   SiteMapper.map('example.com')
   def self.map(source)
     Crawler.collect_urls(source)
   end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: site_mapper
 version: !ruby/object:Gem::Version
-  version: 0.0.3
+  version: 0.0.4
 platform: ruby
 authors:
 - Jacob Burenstam