my_crawler 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 2bf22d64affd3173f46bf53774604346dba33d2c5a164ef960702f82eac13308
+  data.tar.gz: 7e0659ba5a552eca90ac961ff10a162a1f4c81fdfc5c2a11da5888d6988b28a9
+SHA512:
+  metadata.gz: e745ad676a3abede4ed11ae75a5ba0edea849a75d8168ad204f9b59ae8f536275f74ee817b77ac88f1927f3cdda1b2779faa1ee267424bcc6209e8e0c0908bee
+  data.tar.gz: 54173b6140c61cd880cde80c649e4aad59e39e3ed578cc941bf1e328cc19b2436a6ef470b9130cee4cb00cccad7beef546cc83ea37e43ec31a61de82475acb2a

data/bin/crawler

@@ -0,0 +1,8 @@
+#!/usr/bin/env ruby
+
+
+require 'my_crawler'
+
+crawler = MyCrawler.new
+
+crawler.crawl

data/lib/command_line_argument_parser.rb

@@ -0,0 +1,54 @@
+require 'getoptlong'
+
+class Command_line_argument_parser
+    WEB_CRAWLER = 'web'
+    DOMAIN_CRAWLER = 'domain'
+    attr_reader :crawl_type, :crawl_depth, :page_limit, :url_file
+
+  def initialize
+    unless ARGV.length >= 1
+      display_usage
+      exit
+    end
+    @opts = GetoptLong.new(
+      ["--crawl", "-c", GetoptLong::REQUIRED_ARGUMENT],
+      ["--crawl-depth", "-d", GetoptLong::OPTIONAL_ARGUMENT],
+      ["--page-limit", "-p", GetoptLong::OPTIONAL_ARGUMENT],
+      ["--url-file", "-f", GetoptLong::OPTIONAL_ARGUMENT]
+    )
+    @crawl_type = "data.txt"
+    @crawl_depth = 3
+    @page_limit = 100
+    @url_file = 'urls.txt'
+  end
+
+  def display_usage
+    p "Sample usage:"
+    p "ruby search-engine-main.rb -c web -d 3 -p 100 -f 'urls.txt'"
+    p "-c must be either 'web' or 'domain', will default to 'web' if you type garbage"
+  end
+
+  def parse_arguments
+    @opts.each do |opt, arg|
+      case opt
+      when '--crawl'
+        ensure_crawl_type_corrent(arg)
+      when '--crawl-depth'
+        @crawl_depth = arg.to_i
+      when '--page-limit'
+        @page_limit = arg.to_i
+      when '--url-file'
+        @url_file = arg
+      end
+    end
+  end
+
+  def ensure_crawl_type_corrent(value)
+    if value != WEB_CRAWLER && value != DOMAIN_CRAWLER
+      @crawl_type = WEB_CRAWLER
+    else
+      @crawl_type = value
+    end
+  end
+
+end

data/lib/my_crawler.rb

@@ -0,0 +1,27 @@
+require 'spider'
+require 'command_line_argument_parser'
+require 'url_store'
+
+class MyCrawler
+  def initialize
+    @argument_parser = Command_line_argument_parser.new
+    @argument_parser.parse_arguments
+    @spider = Spider.new
+    @url_store = UrlStore.new(@argument_parser.url_file)
+  end
+
+  def crawl
+    if @argument_parser.crawl_type == Command_line_argument_parser::WEB_CRAWLER
+       @spider.crawl_web(
+         @url_store.get_urls,
+         @argument_parser.crawl_depth,
+         @argument_parser.page_limit
+       )
+     else
+      @spider.crawl_domain(
+        @url_store.get_urls,
+        @argument_parser.parse_arguments
+      )
+    end
+  end
+end

data/lib/spider.rb

@@ -0,0 +1,81 @@
+require 'net/http'
+require 'uri'
+require 'open-uri'
+require 'rubygems'
+require 'hpricot'
+require 'url_utils'
+
+class Spider
+  include UrlUtils
+
+  def initialize
+    @already_visited = {}
+  end
+
+  def crawl_web(urls, depth=2, page_limit=100)
+    depth.times do
+      next_urls = []
+      urls.each do |url|
+        url_object = open_url(url)
+        next if url_object.nil?
+
+        url = update_url_if_redirected(url_object)
+        parsed_doc = parse_url(url_object)
+        next if parsed_doc.nil?
+
+        @already_visited[url] = true if @already_visited[url].nil?
+        return if already_visited.size == page_limit
+
+        next_urls += (find_urls_on_page(parsed_doc, url) - @already_visited.keys)
+        next_urls.uniq!
+      end
+      urls = next_urls
+    end
+
+  end
+
+  def crawl_domain(url, page_limit = 100)
+    return if @already_visited.size == page_limit
+
+    url_object = open_url(url)
+    return if url_object.nil?
+
+    parsed_doc = parse_url(url_object)
+    return if parsed_doc.nil?
+
+    @already_visited[url] = true if @already_visited[url].nil?
+    page_urls = find_urls_on_page(parsed_doc, url)
+    page_urls.each do |page_url|
+      if urls_on_same_domain?(url, page_url) && @already_visited[page_url].nil?
+        crawl_domain(page_url)
+      end
+    end
+  end
+
+  def open_url(url)
+    open(url)
+  rescue
+    puts "unable to open url: #{url}"
+  end
+
+  def update_url_if_redirected(url_object)
+    url_object.base_url.to_s
+  end
+
+  def parse_url(url_object)
+    doc = hpricot(url_object)
+    puts "Crawling url " + url_object.base_uri.to_s
+  rescue
+    puts "Could not parse url: " + url_object.base_uri.to_s
+  end
+
+  def find_urls_on_page(parsed_doc, current_url)
+    parsed_doc.search('a[@href]').each_with_object([]) do |x, urls_list|
+      new_url = x['href'].split('#')[0]
+      if new_url
+        new_url = make_absolute(current_url, new_url) if relative?(new_url)
+        urls_list.push(new_url)
+      end
+    end
+  end
+end

data/lib/url_store.rb

@@ -0,0 +1,24 @@
+class UrlStore
+  attr_reader :urls
+  alias :get_urls :urls
+
+  def initialize(url_file)
+    @urls = read_urls_from_file(url_file)
+  end
+
+  def get_url
+    @urls[0]
+  end
+
+  def read_urls_from_file(url_file)
+    urls = []
+    File.open(url_file, 'r') do |file|
+      file.readlines.each do |line|
+        urls.push(line.chomp)
+      end
+    end
+    urls
+  end
+
+  private :read_urls_from_file
+end

data/lib/url_utils.rb

@@ -0,0 +1,70 @@
+module UrlUtils
+  def relative?(url)
+    !url.matach(/^http/)
+  end
+
+  def make_absolute(potential_base, relative_url)
+    if relative_url.match(/^\//)
+      create_absolute_url_from_base(potential_base, relative_url)
+    else
+      create_absolute_url_from_context(potential_base, relative_url)
+    end
+  end
+
+  def urls_on_same_domain?(url1, url2)
+    get_domain(url1) == get_domain(url2)
+  end
+
+  def get_domain(url)
+    remove_extra_paths(url)
+  end
+
+  private
+  def create_absolute_url_from_base(potential_base, relative_url)
+    remove_extra_paths(potential_base) + relative_url
+  end
+
+  def remove_extra_paths(potential_base)
+    index_to_start_slash_search = potential_base.index('://') + 3
+    index_of_first_relevant_slash = potential_base.index('/', index_to_start_slash_search)
+    if index_of_first_relevant_slash != nil?
+      potential_base(0, index_of_first_relevant_slash)
+    else
+      potential_base
+    end
+  end
+
+  def create_absolute_url_from_context(potential_base, relative_url)
+    remove_extra_paths(potential_base) + relative_url
+  end
+
+  def remove_extra_paths(potential_base)
+    index_to_start_slash_search = potential_base.index('://') +3
+    index_of_first_relevant_slash = potential_base.index('/', index_to_start_slash_search)
+    if index_of_first_relevant_slash != nil
+      potential_base[0, index_of_first_relevant_slash]
+    else
+      potential_base
+    end
+  end
+
+  def create_absolute_url_from_context(potential_base, relative_url)
+    absolute_url = nil
+    if potential_base.match(/\/$/)
+      absolute_url = potential_base + relative_url
+    else
+      last_index_of_slash = potential_base.rindex('/')
+      if potential_base[last_index_of_slash-2, 2] == ':/'
+        absolute_url = potential_base + '/' + relative_url
+      else
+        last_index_of_dot = potential_base.rindex(',')
+        if last_index_of_dot < last_index_of_slash
+          absolute_url = potential_base + '/' + relative_url
+        else
+          absolute_url = potential_base[0, last_index_of_slash+1] + relative_url
+        end
+      end
+    end
+    absolute_url
+  end
+end

metadata

@@ -0,0 +1,63 @@
+--- !ruby/object:Gem::Specification
+name: my_crawler
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- 57581600@gmail.com
+autorequire:
+bindir: bin
+cert_chain: []
+date: 2022-10-18 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: hpricot
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.8'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.8'
+description: A simple web crawler gem
+email:
+executables:
+- crawler
+extensions: []
+extra_rdoc_files: []
+files:
+- bin/crawler
+- lib/command_line_argument_parser.rb
+- lib/my_crawler.rb
+- lib/spider.rb
+- lib/url_store.rb
+- lib/url_utils.rb
+homepage: http://rubygems.org/gems/my_crawler
+licenses:
+- MIT
+metadata: {}
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.2.3
+signing_key:
+specification_version: 4
+summary: My first gem
+test_files: []