coolCrawler 0.4.0 → 0.4.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 7aecaddb63af3eb7a621f2cd75472291e0376ca57ef30ec229edcbe274bc704b
-  data.tar.gz: f42a0b9230a6886169cc7dd35f25f03dcb8b1f6d53ec5515c32ed647e2249ee5
+  metadata.gz: 035e8dbe27d4648d16243fea96c692b4a9bde8973d13df02b66aad3aa742afc3
+  data.tar.gz: 687985a3d0e391aedc610ecae0329299a36c8edc90704b9f9d92118532d3d7cc
 SHA512:
-  metadata.gz: 36442036c64c324adf090c7ccb4b6cc474875471ac1dc54ead21604e9de8209036c608e18d0b83bf97e37281720c66cf5b4a8084fb97ed7d81aa7aa1f58e7387
-  data.tar.gz: 38a40361a3ee2d1b34efbf21e482fa4663f89d20731490e1ebd2e6e014cd66537cd95e0da9fdef14ec33d02f59d0aa054fcca383e158e2ed9fbe544501771ce6
+  metadata.gz: 224908b5e8f495063ddc81c178f10e276b2744c8d8b71b16538b3ac2eed70978e686cecc1cbf900d3380a1919ec5e438dd0a1dc559b7d6abbde30e9bd44f819e
+  data.tar.gz: 0ee4aa741315bdd8a2774ee28b762a521f12489356621efe6cf38fab66898a2210313b7e8f07eb74ebd95492b20b80e6548d84a0333f10ac8c09ef6e96410b53

data/lib/coolCrawler/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module CoolCrawler
-  VERSION = "0.4.0"
+  VERSION = "0.4.2"
 end

data/lib/cool_crawler.rb

@@ -13,11 +13,13 @@ module CoolCrawler
   # This is the class that handles the queue and async requests
   class CrawlerPool
 
-    def initialize(start, max_connections, delay)
+    def initialize(start, max_connections, delay, max_pages=50)
       @uri = URI(start)
+      @max_pages = max_pages
       @site = "#{uri.scheme}://#{uri.host}"
       @max_connections = max_connections
       @delay = delay
+      @visited_pages = 0
       visited[uri.path] = 1
       queue << uri.path
     end
@@ -54,6 +56,7 @@ module CoolCrawler
             after(page, links, body)
             links.each do |link|
               enqueue(link)
+              @visited_pages += 1
               add_to_visited(link)
             end
           end
@@ -104,7 +107,18 @@ module CoolCrawler
     end
 
     def enqueue(path)
-      queue << path unless visited.include?(path)
+      unless visited.include?(path) or @visited_pages > @max_pages
+        queue << path
+        p queue.size
+      end
+    end
+
+    def sum_pages
+      sum = 0
+      visited.each do |_k, v|
+        sum += v
+      end
+      sum
     end
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: coolCrawler
 version: !ruby/object:Gem::Version
-  version: 0.4.0
+  version: 0.4.2
 platform: ruby
 authors:
 - William Wright
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2022-10-14 00:00:00.000000000 Z
+date: 2022-10-31 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rspec