coolCrawler 0.2.0 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 2158b8a1d5ceeaeb6d0d7813e6ae5020a49eaffd620453c8490e3e64749fecbb
-  data.tar.gz: 76fcf09ba3252a94a748b6a16ed43787b7fd28f0806aff430a58c2aa884d6266
+  metadata.gz: e927853d47dec36f11557336b9a7c1f2d986b1a9933b78878f2cef18f585cf54
+  data.tar.gz: 67a189af1c21a32650b4cd58573aac781cf2f2a43ce6b20a88eb04a3298fbb79
 SHA512:
-  metadata.gz: 3a51a6c79a067d65e2efb064b0e7aedc9740d5137143cef98d5007d4dfff9a8cd4fde144e8ebca36becf418f01039dfebe037ee1e9f0f658afa624c4ae9f8336
-  data.tar.gz: 3bd9259785c29120037066739c7c55b4b6b011ac7aa252b9f27a82516c8340e90825dd7d868f54e9c49cd7b41ed4804892b1494316adb9bfd3629fa34cbbfffc
+  metadata.gz: 6f3e9b7ff0b17807160670456b7d1bf49079760accef770620e0689e66234f6737dd90c973178f1bc600d22c2bc114e95d198fd756a656f006c2d9f35e2b7167
+  data.tar.gz: 2e03b27c2142c7eb389df2e4955070924c843e92d2a167e6aa1b5411e2698bcfbc551999542575b46eda561a949eaca2a3445a1e7db2ba6304700b58ed8a35b7

data/CHANGELOG.md

@@ -1,4 +1,9 @@
-## [Unreleased]
+## [0.3.0] - 2022-09-24
+
+- Removed Crawler class
+- CrawlerServer has been renamed to CrawlerPool
+- added attr_reader to get site.
+- Crawler still ignores outgoing links for now
 
 ## [0.1.0] - 2022-09-24
 

data/README.md

@@ -23,6 +23,7 @@ Or install it yourself as:
 
 
 ```ruby
+    require 'cool_crawler'
     # create a set of 10 crawlers with a delay of 0.01 seconds between each group of crawl
     crawler = CoolCrawler::CrawlerServer.new("https://github.com", 10, 0.01)
     

data/lib/coolCrawler/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module CoolCrawler
-  VERSION = "0.2.0"
+  VERSION = "0.3.0"
 end

data/lib/cool_crawler.rb

@@ -11,10 +11,10 @@ module CoolCrawler
   class Error < StandardError; end
 
   # This is the class that handles the queue and async requests
-  class CrawlerServer
+  class CrawlerPool
 
     def initialize(start, max_connections, delay)
-      uri = URI(start)
+      @uri = URI(start)
       @site = "#{uri.scheme}://#{uri.host}"
       @max_connections = max_connections
       @delay = delay
@@ -22,7 +22,7 @@ module CoolCrawler
       queue << uri.path
     end
 
-    attr_reader :max_connections, :delay, :callback
+    attr_reader :max_connections, :uri, :delay, :callback, :site
 
     def set_callback(proc)
       @callback=proc
@@ -41,17 +41,15 @@ module CoolCrawler
 
     def send_crawlers
       pages = []
-      until queue.empty? || pages.size >= max_connections
-        pages << queue.pop
-      end
+      pages << queue.pop until queue.empty? || pages.size >= max_connections
       Async do
         internet = Async::HTTP::Internet.new
         barrier = Async::Barrier.new
 
         pages.each do |page|
           barrier.async do
-            response = internet.get URI.join(@site, page)
-            links = Crawler.new(URI.join(@site, page), response.read).gather_links_uri
+            response = internet.get URI.join(@site, page).to_s
+            links = gather_links_uri(response.read, URI.join(uri, page))
             after(page, links)
             links.each do |link|
               enqueue(link)
@@ -65,6 +63,21 @@ module CoolCrawler
       end
     end
 
+    def gather_links_uri(body, page)
+      links = []
+      doc = Nokogiri::HTML(body)
+      doc.xpath("//a").each do |a|
+        next if a["href"].nil?
+        uri_a = URI(a["href"].strip.split('#')[0].sub(/\\|(\s+$)/, ""))
+        begin
+          links << URI.join(page, uri_a).path if (uri_a.host == uri.host || uri_a.host.nil?) && uri_a.path
+        rescue
+          # do nothing
+        end
+      end
+      links
+    end
+
     def queue
       @queue ||= Queue.new
     end
@@ -92,35 +105,5 @@ module CoolCrawler
     def enqueue(path)
       queue << path unless visited.include?(path)
     end
-
-  end
-
-
-  # This is the individual crawler
-  class Crawler
-    include CoolCrawler
-    def initialize(current, response)
-      @current = URI(current)
-      @response = response
-    end
-
-    attr_reader :current, :response
-
-    def gather_links_uri
-      links = []
-      doc = Nokogiri::HTML(response)
-      doc.xpath("//a").each do |a|
-        next if a["href"].nil?
-        uri_a = URI(a["href"].strip.split('#')[0].sub(/\\|(\s+$)/, ""))
-        begin
-          link = URI.join(current, uri_a).path if (uri_a.host == current.host || uri_a.host.nil?) && uri_a.path
-          links << URI.join(current, uri_a).path if (uri_a.host == current.host || uri_a.host.nil?) && uri_a.path
-        rescue
-          # do nothing
-        end
-      end
-      links
-    end
-    
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: coolCrawler
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.3.0
 platform: ruby
 authors:
 - William Wright
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2022-10-03 00:00:00.000000000 Z
+date: 2022-10-05 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rspec