rubyretriever 1.2.0 → 1.2.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 79f0b251e367f085f7b84dd83a10f6a1dfcddd3c
-  data.tar.gz: 0e9b6bc8f66b9efd14921d8f0fc5fddc45042b5f
+  metadata.gz: 752f8125a0153960c3575683d2db334dfd3ad88a
+  data.tar.gz: 96615e58b8d29297184e375747d313e61d6dbf02
 SHA512:
-  metadata.gz: fe1a6c8e118378513c4a4e72adeccc94e212fd5e0d4244f56240830155e42f7b7bde80acdfabbc9fe9ee5b46687bc33d089d73c57690beadee3d04804a9435ac
-  data.tar.gz: 4e6bec31d3416293f2fb72b39cfab5602692a2b9d94cbfb6fba9a653afc6bf718e5eebf84b4a5745d01d2e55959df4e489342b0deca2060e2653bb5b31f4731e
+  metadata.gz: 3a11d2e56f8eba9730f2f36257105bac94767d510029bd1a909055293f2fefe6602d662d871df371cbe7d80eb35f6a9702b5fe6e02d2858665037c0a578524f8
+  data.tar.gz: 82b4a77637f603a809f62c9c1dbc196f6f71c0f6c886d4de6953ad9af9f134f83119320fea556e70d5815c5fae47925407163edb125eee9e4f85c5269a68a896

data/lib/retriever/fetch.rb

@@ -150,8 +150,6 @@ module Retriever
         new_links_arr = process_link_stack
         @temp_link_stack = []
         next if new_links_arr.nil? || new_links_arr.empty?
-        # set operations to see are these in our previous visited pages arr
-        next if new_links_arr.empty?
         @link_stack.concat(new_links_arr)
         next unless @sitemap
         @data.concat(new_links_arr)
@@ -243,6 +241,9 @@ module Retriever
       end
       # empty the stack. most clean way
       @link_stack = []
+      # temp contains redirects + new visitable links
+      # we will re-initialize it as empty right after this function
+      # in the parent method 'async crawl and collect'
       @temp_link_stack.flatten.uniq!
     end
   end

data/lib/retriever/version.rb

@@ -1,4 +1,4 @@
 #
 module Retriever
-  VERSION = '1.2.0'
+  VERSION = '1.2.1'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: rubyretriever
 version: !ruby/object:Gem::Version
-  version: 1.2.0
+  version: 1.2.1
 platform: ruby
 authors:
 - Joe Norton