coolCrawler 0.1.0 → 0.2.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 5ce1b157940f577ce7346cc38f382804ef23475649d46e112c52b2695c1cc35d
-  data.tar.gz: b29f27c7878b1360a1a2f60ce76f67bb751917dbd9b9870ea0922be22a7cb68d
+  metadata.gz: ce0e8b8ba2b209fbe9060b3d00744aaaf574c2b2ada771f1fe4f4bc63665bc4f
+  data.tar.gz: 89f98f2b07274e0d3f4217aa9303899d20a7395ebe3a133980f920087aaa48e4
 SHA512:
-  metadata.gz: e03f00079f407af43f304aa604445a03da5e4bab441fdd5080dfb6531f69fe14781a81434a7acf64a135f5da8c03804f2aac2af3c0cea0fee0f1d25f9546eef1
-  data.tar.gz: 637eb3cff98f1357ccd3ab5095f30c39a120b1348a020083e270ad11ca877e71fa232c1a7bd13f317f138ff7907ad137788812beb25e52e0d7cd4e318c6c6767
+  metadata.gz: 4612e0d29277b25c6e0621efc2b4cfaa0f2d5dbabd94445beef520a3b0d912c78942b73f80731d1394e5a5f9ef5aab51d274188f758bd099a680e2b45297be0f
+  data.tar.gz: 6c06f1eb22dfafbccc8dde7ce245c4d9ed7777ff575742c12a07c88cce1d3bde9b97efc18761bca1a4f2f6d86432cbf74fdbfb8f5ef9b23c69524129449a8236

data/Gemfile.lock

@@ -1,104 +1,102 @@
-PATH
-  remote: .
-  specs:
-    coolCrawler (0.1.0)
-      mechanize
-      nokogiri (~> 1.13)
-      open-uri
-      uri
-
-GEM
-  remote: https://rubygems.org/
-  specs:
-    addressable (2.8.1)
-      public_suffix (>= 2.0.2, < 6.0)
-    ast (2.4.2)
-    connection_pool (2.3.0)
-    date (3.2.2)
-    diff-lcs (1.5.0)
-    domain_name (0.5.20190701)
-      unf (>= 0.0.5, < 1.0.0)
-    http-cookie (1.0.5)
-      domain_name (~> 0.5)
-    json (2.6.2)
-    mechanize (2.8.5)
-      addressable (~> 2.8)
-      domain_name (~> 0.5, >= 0.5.20190701)
-      http-cookie (~> 1.0, >= 1.0.3)
-      mime-types (~> 3.0)
-      net-http-digest_auth (~> 1.4, >= 1.4.1)
-      net-http-persistent (>= 2.5.2, < 5.0.dev)
-      nokogiri (~> 1.11, >= 1.11.2)
-      rubyntlm (~> 0.6, >= 0.6.3)
-      webrick (~> 1.7)
-      webrobots (~> 0.1.2)
-    mime-types (3.4.1)
-      mime-types-data (~> 3.2015)
-    mime-types-data (3.2022.0105)
-    net-http-digest_auth (1.4.1)
-    net-http-persistent (4.0.1)
-      connection_pool (~> 2.2)
-    nokogiri (1.13.8-x64-mingw-ucrt)
-      racc (~> 1.4)
-    open-uri (0.2.0)
-      stringio
-      time
-      uri
-    parallel (1.22.1)
-    parser (3.1.2.1)
-      ast (~> 2.4.1)
-    public_suffix (5.0.0)
-    racc (1.6.0)
-    rainbow (3.1.1)
-    rake (13.0.6)
-    regexp_parser (2.5.0)
-    rexml (3.2.5)
-    rspec (3.11.0)
-      rspec-core (~> 3.11.0)
-      rspec-expectations (~> 3.11.0)
-      rspec-mocks (~> 3.11.0)
-    rspec-core (3.11.0)
-      rspec-support (~> 3.11.0)
-    rspec-expectations (3.11.1)
-      diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.11.0)
-    rspec-mocks (3.11.1)
-      diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.11.0)
-    rspec-support (3.11.1)
-    rubocop (1.36.0)
-      json (~> 2.3)
-      parallel (~> 1.10)
-      parser (>= 3.1.2.1)
-      rainbow (>= 2.2.2, < 4.0)
-      regexp_parser (>= 1.8, < 3.0)
-      rexml (>= 3.2.5, < 4.0)
-      rubocop-ast (>= 1.20.1, < 2.0)
-      ruby-progressbar (~> 1.7)
-      unicode-display_width (>= 1.4.0, < 3.0)
-    rubocop-ast (1.21.0)
-      parser (>= 3.1.1.0)
-    ruby-progressbar (1.11.0)
-    rubyntlm (0.6.3)
-    stringio (3.0.2)
-    time (0.2.0)
-      date
-    unf (0.1.4)
-      unf_ext
-    unf_ext (0.0.8.2-x64-mingw-ucrt)
-    unicode-display_width (2.3.0)
-    uri (0.11.0)
-    webrick (1.7.0)
-    webrobots (0.1.2)
-
-PLATFORMS
-  x64-mingw-ucrt
-
-DEPENDENCIES
-  coolCrawler!
-  rake (~> 13.0)
-  rspec (~> 3.0)
-  rubocop (~> 1.21)
-
-BUNDLED WITH
-   2.3.7
+PATH
+  remote: .
+  specs:
+    coolCrawler (0.1.0)
+      async-http
+      nokogiri
+      open-uri
+      uri
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    ast (2.4.2)
+    async (2.1.0)
+      console (~> 1.10)
+      io-event (~> 1.0.0)
+      timers (~> 4.1)
+    async-http (0.59.2)
+      async (>= 1.25)
+      async-io (>= 1.28)
+      async-pool (>= 0.2)
+      protocol-http (~> 0.23.1)
+      protocol-http1 (~> 0.14.0)
+      protocol-http2 (~> 0.14.0)
+      traces (>= 0.4.0)
+    async-io (1.34.0)
+      async
+    async-pool (0.3.12)
+      async (>= 1.25)
+    console (1.15.3)
+      fiber-local
+    date (3.2.2)
+    diff-lcs (1.5.0)
+    fiber-local (1.0.0)
+    io-event (1.0.9)
+    json (2.6.2)
+    nokogiri (1.13.8-x86_64-linux)
+      racc (~> 1.4)
+    open-uri (0.2.0)
+      stringio
+      time
+      uri
+    parallel (1.22.1)
+    parser (3.1.2.1)
+      ast (~> 2.4.1)
+    protocol-hpack (1.4.2)
+    protocol-http (0.23.12)
+    protocol-http1 (0.14.6)
+      protocol-http (~> 0.22)
+    protocol-http2 (0.14.2)
+      protocol-hpack (~> 1.4)
+      protocol-http (~> 0.18)
+    racc (1.6.0)
+    rainbow (3.1.1)
+    rake (13.0.6)
+    regexp_parser (2.6.0)
+    rexml (3.2.5)
+    rspec (3.11.0)
+      rspec-core (~> 3.11.0)
+      rspec-expectations (~> 3.11.0)
+      rspec-mocks (~> 3.11.0)
+    rspec-core (3.11.0)
+      rspec-support (~> 3.11.0)
+    rspec-expectations (3.11.1)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.11.0)
+    rspec-mocks (3.11.1)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.11.0)
+    rspec-support (3.11.1)
+    rubocop (1.36.0)
+      json (~> 2.3)
+      parallel (~> 1.10)
+      parser (>= 3.1.2.1)
+      rainbow (>= 2.2.2, < 4.0)
+      regexp_parser (>= 1.8, < 3.0)
+      rexml (>= 3.2.5, < 4.0)
+      rubocop-ast (>= 1.20.1, < 2.0)
+      ruby-progressbar (~> 1.7)
+      unicode-display_width (>= 1.4.0, < 3.0)
+    rubocop-ast (1.21.0)
+      parser (>= 3.1.1.0)
+    ruby-progressbar (1.11.0)
+    stringio (3.0.2)
+    time (0.2.0)
+      date
+    timers (4.3.4)
+    traces (0.7.0)
+    unicode-display_width (2.3.0)
+    uri (0.11.0)
+
+PLATFORMS
+  x86_64-linux
+
+DEPENDENCIES
+  coolCrawler!
+  rake (~> 13.0)
+  rspec (~> 3.0)
+  rubocop (~> 1.21)
+
+BUNDLED WITH
+   2.3.7

data/README.md

@@ -1,8 +1,6 @@
 # CoolCrawler
 
-Welcome to your new gem! In this directory, you'll find the files you need to be able to package up your Ruby library into a gem. Put your Ruby code in the file `lib/coolCrawler`. To experiment with that code, run `bin/console` for an interactive prompt.
-
-TODO: Delete this and the text above, and describe your gem
+Cool Crawler is a light weight crawler for the purpose of that one university assignment.
 
 ## Installation
 
@@ -22,7 +20,31 @@ Or install it yourself as:
 
 ## Usage
 
-TODO: Write usage instructions here
+
+
+```ruby
+    # create a set of 10 crawlers with a delay of 0.01 seconds between each group of crawl
+    crawler = CoolCrawler::CrawlerServer.new("https://github.com", 10, 0.01)
+    
+    # set the callback function. This will be called everytime an individual crawler finishes its crawling. page is current path the crawler is on, links is an array
+    # of all links found
+    crawler.set_callback(Proc.new {|page, links| p page, links})
+
+    # starts the crawl (ends when thare no more page in queue)
+    crawler.run
+    end
+```
+
+## TO-DO
+
+### For version 0.1.x:    
+
+* Implement method to scan and apply the rules of robots.txt 
+* add a way to limit the number of links in the queue 
+* Sleep period can be specified in the start block, but it would be good to have it supplied as configuration 
+* Test and adapt for concurrency 
+
+I will implement pageranking in 0.2.x   
 
 ## Development
 

data/coolCrawler.gemspec

@@ -31,7 +31,8 @@ Gem::Specification.new do |spec|
   # Uncomment to register a new dependency of your gem
   # spec.add_dependency "example-gem", "~> 1.0"
   spec.add_development_dependency "rspec", "~> 3.11"
-  spec.add_dependency "nokogiri", "~> 1.13"
+  spec.add_dependency 'async-http'
+  spec.add_dependency "nokogiri"
   spec.add_dependency "open-uri"
   spec.add_dependency "uri"
   # For more information and examples about making a new gem, check out our

data/lib/coolCrawler/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module CoolCrawler
-  VERSION = "0.1.0"
+  VERSION = "0.2.1"
 end

data/lib/cool_crawler.rb

@@ -1,98 +1,126 @@
 # frozen_string_literal: true
 
 require_relative "coolCrawler/version"
-require "Nokogiri"
-require "net/http"
-require "open-uri"
-require "uri"
+require 'async'
+require 'async/http/internet'
+require 'async/barrier'
+require 'nokogiri'
+
 # Module Controller
 module CoolCrawler
   class Error < StandardError; end
 
-  # Main Crawler Class
-  class Crawler
-    include CoolCrawler
-    def initialize(root)
-      @uri = URI(root)
-      @website = "#{@uri.scheme}://#{@uri.host}"
-      @queue = []
-      @visited = { @uri.path => 1 }
-      @queue << @uri.path # root is part of the pages to crawl
+  # This is the class that handles the queue and async requests
+  class CrawlerServer
+
+    def initialize(start, max_connections, delay)
+      uri = URI(start)
+      @site = "#{uri.scheme}://#{uri.host}"
+      @max_connections = max_connections
+      @delay = delay
+      visited[uri.path] = 1
+      queue << uri.path
     end
 
-    attr_reader :queue, :visted, :uri, :website
+    attr_reader :max_connections, :delay, :callback
 
-    def http
-      @http ||= Net::HTTP.new(@uri.host)
+    def set_callback(proc)
+      @callback=proc
     end
 
-    def start
-      return unless block_given?
+    def run
+      until queue.empty?
+        send_crawlers
+        sleep(delay)
+      end
+    end
 
-      yield(self.next) until queue.empty?
+    def after(page, links)
+      callback.call(page, links) unless callback.nil?
     end
 
-    def unvisited?(path)
-      !@visited.include?(path)
+    def send_crawlers
+      pages = []
+      until queue.empty? || pages.size >= max_connections
+        pages << queue.pop
+      end
+      Async do
+        internet = Async::HTTP::Internet.new
+        barrier = Async::Barrier.new
+
+        pages.each do |page|
+          barrier.async do
+            response = internet.get URI.join(@site, page).to_s
+            links = Crawler.new(URI.join(@site, page), response.read).gather_links_uri
+            after(page, links)
+            links.each do |link|
+              enqueue(link)
+              add_to_visited(link)
+            end
+          end
+        end
+        barrier.wait
+      ensure
+        internet&.close
+      end
+    end
+
+    def queue
+      @queue ||= Queue.new
+    end
+
+    def visited
+      @visited ||= {}
     end
 
     def visited?(path)
-      @visited.include?(path)
+      visited.include?(path)
     end
 
     def add_to_visited(path)
       if visited?(path)
-        @visited[path] += 1
+        visited[path] += 1
       else
-        @visited[path] = 1
+        visited[path] = 1
       end
     end
 
-    def all_links
-      @visited.sort_by { |_k, v| v }
+    def sorted_visited
+      visited.sort_by { |_k, v| v }
     end
 
-    private
+    def enqueue(path)
+      queue << path unless visited.include?(path)
+    end
 
-    def gather_links_uri(doc)
-      doc.xpath("//a").each do |a|
-        next if a["href"].nil?
+  end
 
-        uri_a = URI(a["href"])
-        yield URI.join(@current, uri_a).path if (uri_a.host == @uri.host || uri_a.host.nil?) && uri_a.path
-      end
-    end
 
-    def get_body(uri)
-      http.read_timeout = 5
-      http.max_retries = 5
-      res = http.get(uri.path)
-      res.body
+  # This is the individual crawler
+  class Crawler
+    include CoolCrawler
+    def initialize(current, response)
+      @current = URI(current)
+      @response = response
     end
 
-    def enqueue(path)
-      queue << path unless @visited.include?(path)
-    end
+    attr_reader :current, :response
 
-    # Gathers the a['href'] links on a page and enqueues them
-    def add_to_queue(path)
+    def gather_links_uri
       links = []
-      @current = URI.join(@website, path)
-      doc = Nokogiri::HTML(get_body(@current))
-      gather_links_uri(doc) do |link_uri|
-        enqueue(link_uri)
-        add_to_visited(link_uri)
-        links << link_uri
+      doc = Nokogiri::HTML(response)
+      doc.xpath("//a").each do |a|
+        next if a["href"].nil?
+        uri_a = URI(a["href"].strip.split('#')[0].sub(/\\|(\s+$)/, ""))
+        begin
+          link = URI.join(current, uri_a).path if (uri_a.host == current.host || uri_a.host.nil?) && uri_a.path
+          links << URI.join(current, uri_a).path if (uri_a.host == current.host || uri_a.host.nil?) && uri_a.path
+        rescue
+          # do nothing
+        end
       end
-      [path, links]
-    end
-
-    def visit(path)
-      add_to_queue(path)
-    end
-
-    def next
-      visit(@queue.pop)
+      links
     end
+    
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: coolCrawler
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.2.1
 platform: ruby
 authors:
 - William Wright
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2022-09-29 00:00:00.000000000 Z
+date: 2022-10-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rspec
@@ -24,20 +24,34 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '3.11'
+- !ruby/object:Gem::Dependency
+  name: async-http
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: nokogiri
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '1.13'
+        version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '1.13'
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: open-uri
   requirement: !ruby/object:Gem::Requirement
@@ -82,6 +96,7 @@ files:
 - LICENSE.txt
 - README.md
 - Rakefile
+- coolCrawler-0.2.0.gem
 - coolCrawler.gemspec
 - lib/coolCrawler/version.rb
 - lib/cool_crawler.rb