site_mapper 0.0.5 → 0.0.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 2bab1dcdbee3e862d0b7a5ac04e5e6995742ec37
-  data.tar.gz: df5e1bb8503c30d32f446dc9840412171eba4eb5
+  metadata.gz: 6870976b0c732915cd3077c0be8bbfd6f92d7f45
+  data.tar.gz: 1f2949d91774c87dd2f156b562ffb9e532f924fe
 SHA512:
-  metadata.gz: 20c921483737e916b0009598124c03bc29f5ba919fa76b718ac6bf84550903eba542aee2e37eda3a6411cef2c0da7982809235454d7d0b7447c09c6fad5816d9
-  data.tar.gz: 81e0f10b5e39c355b0a3be35742aec7c1cda6d1a6adf3de6113d41ef1706cdcaa0a87eec87eb473fc7cb3d8b5e2ce3abfbf70521205ed01f4c7c39c01f0b3c95
+  metadata.gz: d736d6c6d62c55e9f38417bb9061dfb19de907ef00d2e41cb31bd1d190d92fd16b319d02483673c333cfcc6cf6feed720906b2ba914957f13f602fc595ef2223
+  data.tar.gz: 69661165ac90ede2611f2d24ba43c58c1eed7b17c5405b23d00df0de023e4896210f75ab152e8d2311832ee4aa828d17c59887e0e943e00b8a0daa34d2d3b1ad

data/lib/site_mapper/crawl_url.rb

@@ -4,7 +4,7 @@ module SiteMapper
     attr_reader :resolved_base_url, :base_hostname
 
     def initialize(base_url)
-      @resolved_base_url = Request.resolve_url(base_url)
+      @resolved_base_url = Request.resolve_url(base_url, with_query: false)
       @base_hostname     = URI.parse(@resolved_base_url).hostname
       @resolved_base_url.prepend('http://') unless @resolved_base_url.start_with?('http')
     end

data/lib/site_mapper/crawler.rb

@@ -12,9 +12,8 @@ module SiteMapper
       base_url     = Request.resolve_url(url)
       @options     = { resolve: resolve }
       @crawl_url   = CrawlUrl.new(base_url)
-      @fetch_queue = Set.new
+      @fetch_queue = CrawlQueue.new
       @processed   = Set.new
-      @fetch_queue << @crawl_url.resolved_base_url
     end
 
     # @see #collect_urls
@@ -27,15 +26,15 @@ module SiteMapper
     # @example URLs for example.com
     #   crawler = Crawler.new('example.com')
     #   crawler.collect_urls
-    # @example URLs for example.com with block
+    # @example URLs for example.com with block (executes in its own thread)
     #   crawler = Crawler.new('example.com')
     #   crawler.collect_urls do |new_url|
     #     puts "New URL found: #{new_url}"
     #   end
     def collect_urls
+      @fetch_queue << @crawl_url.resolved_base_url
       until @fetch_queue.empty?
-        url = @fetch_queue.first
-        @fetch_queue.delete(@fetch_queue.first)
+        url = @fetch_queue.pop
         Thread.new { yield(url) if block_given? }
         page_links(url)
       end
@@ -65,4 +64,18 @@ module SiteMapper
       @options[:resolve] ? Request.resolve_url(url) : url
     end
   end
+
+  class CrawlQueue
+    def self.new
+      Set.new.extend(EnumerablePop)
+    end
+    
+    module EnumerablePop
+      def pop
+        first_element = first
+        delete(first_element)
+        first_element
+      end
+    end
+  end
 end

data/lib/site_mapper/request.rb

@@ -21,8 +21,15 @@ module SiteMapper
         http.request(request)
       end
 
-      def resolve_url(url)
-        UrlResolver.resolve(url)
+      def resolve_url(url, with_query: true)
+        resolved = UrlResolver.resolve(url)
+        resolved = remove_query(resolved) unless with_query
+        resolved
+      end
+
+      def remove_query(url)
+        index = url.index('?')
+        index.nil? ? url : url[0...index]
       end
     end
   end

data/lib/site_mapper/version.rb

@@ -1,3 +1,3 @@
 module SiteMapper
-  VERSION = '0.0.5'
+  VERSION = '0.0.6'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: site_mapper
 version: !ruby/object:Gem::Version
-  version: 0.0.5
+  version: 0.0.6
 platform: ruby
 authors:
 - Jacob Burenstam