coolCrawler 0.3.0 → 0.4.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: e927853d47dec36f11557336b9a7c1f2d986b1a9933b78878f2cef18f585cf54
-  data.tar.gz: 67a189af1c21a32650b4cd58573aac781cf2f2a43ce6b20a88eb04a3298fbb79
+  metadata.gz: 035e8dbe27d4648d16243fea96c692b4a9bde8973d13df02b66aad3aa742afc3
+  data.tar.gz: 687985a3d0e391aedc610ecae0329299a36c8edc90704b9f9d92118532d3d7cc
 SHA512:
-  metadata.gz: 6f3e9b7ff0b17807160670456b7d1bf49079760accef770620e0689e66234f6737dd90c973178f1bc600d22c2bc114e95d198fd756a656f006c2d9f35e2b7167
-  data.tar.gz: 2e03b27c2142c7eb389df2e4955070924c843e92d2a167e6aa1b5411e2698bcfbc551999542575b46eda561a949eaca2a3445a1e7db2ba6304700b58ed8a35b7
+  metadata.gz: 224908b5e8f495063ddc81c178f10e276b2744c8d8b71b16538b3ac2eed70978e686cecc1cbf900d3380a1919ec5e438dd0a1dc559b7d6abbde30e9bd44f819e
+  data.tar.gz: 0ee4aa741315bdd8a2774ee28b762a521f12489356621efe6cf38fab66898a2210313b7e8f07eb74ebd95492b20b80e6548d84a0333f10ac8c09ef6e96410b53

data/lib/coolCrawler/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module CoolCrawler
-  VERSION = "0.3.0"
+  VERSION = "0.4.2"
 end

data/lib/cool_crawler.rb

@@ -13,11 +13,13 @@ module CoolCrawler
   # This is the class that handles the queue and async requests
   class CrawlerPool
 
-    def initialize(start, max_connections, delay)
+    def initialize(start, max_connections, delay, max_pages=50)
       @uri = URI(start)
+      @max_pages = max_pages
       @site = "#{uri.scheme}://#{uri.host}"
       @max_connections = max_connections
       @delay = delay
+      @visited_pages = 0
       visited[uri.path] = 1
       queue << uri.path
     end
@@ -35,8 +37,8 @@ module CoolCrawler
       end
     end
 
-    def after(page, links)
-      callback.call(page, links) unless callback.nil?
+    def after(page, links, body)
+      callback.call(page, links, body) unless callback.nil?
     end
 
     def send_crawlers
@@ -49,10 +51,12 @@ module CoolCrawler
         pages.each do |page|
           barrier.async do
             response = internet.get URI.join(@site, page).to_s
-            links = gather_links_uri(response.read, URI.join(uri, page))
-            after(page, links)
+            body = response.read
+            links = gather_links_uri(body, URI.join(uri, page))
+            after(page, links, body)
             links.each do |link|
               enqueue(link)
+              @visited_pages += 1
               add_to_visited(link)
             end
           end
@@ -103,7 +107,18 @@ module CoolCrawler
     end
 
     def enqueue(path)
-      queue << path unless visited.include?(path)
+      unless visited.include?(path) or @visited_pages > @max_pages
+        queue << path
+        p queue.size
+      end
+    end
+
+    def sum_pages
+      sum = 0
+      visited.each do |_k, v|
+        sum += v
+      end
+      sum
     end
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: coolCrawler
 version: !ruby/object:Gem::Version
-  version: 0.3.0
+  version: 0.4.2
 platform: ruby
 authors:
 - William Wright
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2022-10-05 00:00:00.000000000 Z
+date: 2022-10-31 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rspec