coolCrawler 0.4.0 → 0.4.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 7aecaddb63af3eb7a621f2cd75472291e0376ca57ef30ec229edcbe274bc704b
-  data.tar.gz: f42a0b9230a6886169cc7dd35f25f03dcb8b1f6d53ec5515c32ed647e2249ee5
+  metadata.gz: a8c2944180ee7c5d7f1241fd62bf5e89973a2fca742c51567e1550f14f95f70a
+  data.tar.gz: 6635f39af2babaead932e94cae419738d7bd61e0ecc2f23216cb6102607904bf
 SHA512:
-  metadata.gz: 36442036c64c324adf090c7ccb4b6cc474875471ac1dc54ead21604e9de8209036c608e18d0b83bf97e37281720c66cf5b4a8084fb97ed7d81aa7aa1f58e7387
-  data.tar.gz: 38a40361a3ee2d1b34efbf21e482fa4663f89d20731490e1ebd2e6e014cd66537cd95e0da9fdef14ec33d02f59d0aa054fcca383e158e2ed9fbe544501771ce6
+  metadata.gz: 4f56da51ab47060e7d58b3ac469ca15501b2b457a936b634b361afa0f832a27130c74169d7c1da4f3a188f56db58335c59325de261990ee02bbdbd1431ad24da
+  data.tar.gz: 77dc4e5cdb5d0098ba8f91279bc4d658c653130313fbb49de52c3e2c2b707c4501fcf77a31f2a6a54b0b48ca12e54ab674423ab377e22e043a6e6cce0093a6df

data/lib/coolCrawler/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module CoolCrawler
-  VERSION = "0.4.0"
+  VERSION = "0.4.3"
 end

data/lib/cool_crawler.rb

@@ -13,11 +13,13 @@ module CoolCrawler
   # This is the class that handles the queue and async requests
   class CrawlerPool
 
-    def initialize(start, max_connections, delay)
+    def initialize(start, max_connections, delay, max_pages=50)
       @uri = URI(start)
+      @max_pages = max_pages
       @site = "#{uri.scheme}://#{uri.host}"
       @max_connections = max_connections
       @delay = delay
+      @visited_pages = 0
       visited[uri.path] = 1
       queue << uri.path
     end
@@ -49,9 +51,12 @@ module CoolCrawler
         pages.each do |page|
           barrier.async do
             response = internet.get URI.join(@site, page).to_s
-            body = response.read
-            links = gather_links_uri(body, URI.join(uri, page))
-            after(page, links, body)
+            body = Nokogiri::HTML(response.read)
+            body.search('//img').remove
+            body.search('//style').remove
+            body.search('//script').remove
+            links = gather_links_uri(body.to_s, URI.join(uri, page))
+            after(page, links, body.to_s)
             links.each do |link|
               enqueue(link)
               add_to_visited(link)
@@ -71,7 +76,9 @@ module CoolCrawler
         next if a["href"].nil?
         uri_a = URI(a["href"].strip.split('#')[0].sub(/\\|(\s+$)/, ""))
         begin
-          links << URI.join(page, uri_a).path if (uri_a.host == uri.host || uri_a.host.nil?) && uri_a.path
+          if @visited_pages <= @max_pages
+            links << URI.join(page, uri_a).path if (uri_a.host == uri.host || uri_a.host.nil?) && uri_a.path
+          end
         rescue
           # do nothing
         end
@@ -97,6 +104,7 @@ module CoolCrawler
       else
         visited[path] = 1
       end
+      @visited_pages += 1
     end
 
     def sorted_visited
@@ -104,7 +112,17 @@ module CoolCrawler
     end
 
     def enqueue(path)
-      queue << path unless visited.include?(path)
+      unless visited.include?(path)
+        queue << path
+      end
+    end
+
+    def sum_pages
+      sum = 0
+      visited.each do |_k, v|
+        sum += v
+      end
+      sum
     end
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: coolCrawler
 version: !ruby/object:Gem::Version
-  version: 0.4.0
+  version: 0.4.3
 platform: ruby
 authors:
 - William Wright
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2022-10-14 00:00:00.000000000 Z
+date: 2022-11-01 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rspec