coolCrawler 0.4.2 → 0.4.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 035e8dbe27d4648d16243fea96c692b4a9bde8973d13df02b66aad3aa742afc3
-  data.tar.gz: 687985a3d0e391aedc610ecae0329299a36c8edc90704b9f9d92118532d3d7cc
+  metadata.gz: a8c2944180ee7c5d7f1241fd62bf5e89973a2fca742c51567e1550f14f95f70a
+  data.tar.gz: 6635f39af2babaead932e94cae419738d7bd61e0ecc2f23216cb6102607904bf
 SHA512:
-  metadata.gz: 224908b5e8f495063ddc81c178f10e276b2744c8d8b71b16538b3ac2eed70978e686cecc1cbf900d3380a1919ec5e438dd0a1dc559b7d6abbde30e9bd44f819e
-  data.tar.gz: 0ee4aa741315bdd8a2774ee28b762a521f12489356621efe6cf38fab66898a2210313b7e8f07eb74ebd95492b20b80e6548d84a0333f10ac8c09ef6e96410b53
+  metadata.gz: 4f56da51ab47060e7d58b3ac469ca15501b2b457a936b634b361afa0f832a27130c74169d7c1da4f3a188f56db58335c59325de261990ee02bbdbd1431ad24da
+  data.tar.gz: 77dc4e5cdb5d0098ba8f91279bc4d658c653130313fbb49de52c3e2c2b707c4501fcf77a31f2a6a54b0b48ca12e54ab674423ab377e22e043a6e6cce0093a6df

data/lib/coolCrawler/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module CoolCrawler
-  VERSION = "0.4.2"
+  VERSION = "0.4.3"
 end

data/lib/cool_crawler.rb

@@ -51,12 +51,14 @@ module CoolCrawler
         pages.each do |page|
           barrier.async do
             response = internet.get URI.join(@site, page).to_s
-            body = response.read
-            links = gather_links_uri(body, URI.join(uri, page))
-            after(page, links, body)
+            body = Nokogiri::HTML(response.read)
+            body.search('//img').remove
+            body.search('//style').remove
+            body.search('//script').remove
+            links = gather_links_uri(body.to_s, URI.join(uri, page))
+            after(page, links, body.to_s)
             links.each do |link|
               enqueue(link)
-              @visited_pages += 1
               add_to_visited(link)
             end
           end
@@ -74,7 +76,9 @@ module CoolCrawler
         next if a["href"].nil?
         uri_a = URI(a["href"].strip.split('#')[0].sub(/\\|(\s+$)/, ""))
         begin
-          links << URI.join(page, uri_a).path if (uri_a.host == uri.host || uri_a.host.nil?) && uri_a.path
+          if @visited_pages <= @max_pages
+            links << URI.join(page, uri_a).path if (uri_a.host == uri.host || uri_a.host.nil?) && uri_a.path
+          end
         rescue
           # do nothing
         end
@@ -100,6 +104,7 @@ module CoolCrawler
       else
         visited[path] = 1
       end
+      @visited_pages += 1
     end
 
     def sorted_visited
@@ -107,9 +112,8 @@ module CoolCrawler
     end
 
     def enqueue(path)
-      unless visited.include?(path) or @visited_pages > @max_pages
+      unless visited.include?(path)
         queue << path
-        p queue.size
       end
     end
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: coolCrawler
 version: !ruby/object:Gem::Version
-  version: 0.4.2
+  version: 0.4.3
 platform: ruby
 authors:
 - William Wright
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2022-10-31 00:00:00.000000000 Z
+date: 2022-11-01 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rspec