dk_crawler 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 3f922703479939e3e4aeb66cc09e3eff2ab3a99c0ab53ad270f535c27df367e0
+  data.tar.gz: 57eda2be60eba639be8bd7927be6cf5ca0a21c68a40a45f3bd01f87df986f726
+SHA512:
+  metadata.gz: 875908fe8cd7574fccae8ea1ea566cb36b20f756de7e3c2e2ab31572c1c7b8a58972a0cacf827c3f1cebac1d98a8c399e1f354391457479545294aeaa7d34289
+  data.tar.gz: a605666db11802894ea52b4a4503df9383eff1cc2eec07488ab31e953febdf656df081511a1a46a13e78d0d3359e9aa3b6503e30de4c69735d5b0f7a7ea6b422

data/bin/crawler

@@ -0,0 +1,5 @@
+#!usr/bin/env ruby
+require 'dk_crawler'
+
+crawler = DkCrawler.new
+crawler.crawl

data/lib/command_line_argument_parser.rb

@@ -0,0 +1,53 @@
+require 'getoptlong'
+
+class CommandLineArgumentParser
+  WEB_CRAWLER = 'web'
+  DOMAIN_CRAWLER = 'domain'
+  attr_reader :crawl_type, :crawl_depth, :page_limit, :url_file
+
+  def initialize
+    unless ARGV.length >= 1
+      display_usage
+      exit
+    end
+    @opts = GetoptLong.new(
+      ["--crawl", "-c", GetoptLong::REQUIRED_ARGUMENT],
+      ["--crawl-depth", "-d", GetoptLong::OPTIONAL_ARGUMENT],
+      ["--page-limit", "-p", GetoptLong::OPTIONAL_ARGUMENT],
+      ["--url-file", "-f", GetoptLong::OPTIONAL_ARGUMENT]
+    )
+    @crawl_type = "data.txt"
+    @crawl_depth = 3
+    @page_limit = 100
+    @url_file = 'urls.txt'
+  end
+
+  def display_usage
+    p "Sample usage:"
+    p "ruby dk_crawler.rb -c web -d 3 -p 100 -f 'urls.txt'"
+    p "-c must be either 'web' or 'domain', will default to 'web' is you type garbage "
+  end
+
+  def parse_arguments
+    @opts.each do |opt, arg|
+      case opt
+      when '--crawl'
+        ensure_crawl_type_correct(arg)
+      when '--crawl-depth'
+        @crawl_depth = arg.to_i
+      when '--page-limit'
+        @page_limit = arg.to_i
+      when '--url-file'
+        @url_file = arg
+      end
+    end
+  end
+
+  def ensure_crawl_type_correct(value)
+    if value != WEB_CRAWLER && value != DOMAIN_CRAWLER
+      @crawl_type = WEB_CRAWLER
+    else
+      @crawl_type = value
+    end
+  end
+end

data/lib/dk_crawler.rb

@@ -0,0 +1,27 @@
+require 'spider'
+require 'command_line_argument_parser'
+require 'url_store'
+
+class DkCrawler
+  def initialize
+    @argument_parser = CommandLineArgumentParser.new
+    @argument_parser.parse_arguments
+    @spider = Spider.new
+    @url_store = UrlStore.new(@argument_parser.url_file)
+  end
+
+  def crawl
+    if @argument_parser.crawl_type == CommandLineArgumentParser::WEB_CRAWLER
+      @spider.crawl_web(
+        @url_store.get_urls,
+        @argument_parser.crawl_depth,
+        @argument_parser.page_limit
+      )
+    else
+      @spider.crawl_domain(
+        @url_store.get_url,
+        @argument_parser.page_limit
+      )
+    end
+  end
+end

data/lib/spider.rb

@@ -0,0 +1,84 @@
+require 'net/http'
+require 'uri'
+require 'open-uri'
+require 'nokogiri'
+require 'url_utils'
+
+class Spider
+  include UrlUtils
+  def initialize
+    @already_visited = {}
+  end
+
+  def crawl_web(urls, depth=2, page_limit = 100)
+    depth.times do
+      next_urls = []
+      urls.each do |url|
+        url_object = open_url(url)
+
+        next if url_object.nil?
+
+        url = update_url_if_redirected(url_object)
+        parsed_doc = parse_url(url_object)
+        next if parsed_doc.nil?
+
+        @already_visited[url] = true if @already_visited[url].nil?
+        return if @already_visited.size == page_limit
+
+        next_urls += (find_urls_on_page(parsed_doc, url) - @already_visited.keys)
+        next_urls.uniq!
+      end
+      urls = next_urls
+    end
+  end
+
+  def crawl_domain(url, page_limit = 100)
+    return if @already_visited.size == page_limit
+
+    url_object = open_url(url)
+    return if url_object.nil?
+
+    parsed_doc = parse_url(url_object)
+    return if parsed_doc.nil?
+
+    @already_visited[url] = true if @already_visited[url].nil?
+    page_urls = find_urls_on_page(parsed_doc, url)
+    page_urls.each do |page_url|
+      if urls_on_same_domain?(url, page_url) && @already_visited[page_url].nil?
+        crawl_domain(page_url)
+      end
+    end
+  end
+
+  private
+
+  def open_url(url)
+    open(url)
+  rescue
+    puts "Unable to open url: " + url
+  end
+
+  def update_url_if_redirected(url_object)
+    url_object.base_uri.to_s
+  end
+
+  def parse_url(url_object)
+    doc = Nokogiri::HTML(url_object) #nokogiri
+    puts 'Crawling url ' + url_object.base_uri.to_s
+    doc
+  rescue
+    puts 'Could not parse url: ' + url_object.base_uri.to_s
+  end
+
+  def find_urls_on_page(parsed_doc, current_url)
+    parsed_doc.search('a[@href]').each_with_object([]) do |x, urls_list|
+      new_url = x['href'].split('#')[0]
+      if new_url
+        # complicated feature: make_absolute
+        new_url = make_absolute(current_url, new_url) if relative?(new_url)
+        urls_list.push(new_url)
+
+      end
+    end
+  end
+end

data/lib/url_store.rb

@@ -0,0 +1,24 @@
+class UrlStore
+  attr_reader :urls
+  alias :get_urls :urls
+
+  def initialize(url_file)
+    @urls = read_urls_from_file(url_file)
+  end
+
+  def get_url
+    @urls
+  end
+
+  def read_urls_from_file(url_file)
+    urls = []
+    File.open(url_file, 'r') do |file|
+      file.readlines.each do |line|
+        urls.push(line.chomp)
+      end
+    end
+    urls
+  end
+
+  private :read_urls_from_file
+end

data/lib/url_utils.rb

@@ -0,0 +1,57 @@
+module UrlUtils
+  def relative?(url)
+    !url.match(/^http/)
+  end
+
+  def make_absolute(potential_base, relative_url)
+    if relative_url.match(/^\//)
+      create_absolute_url_from_base(potential_base, relative_url)
+    else
+      create_absolute_url_from_context(potential_base, relative_url)
+    end
+  end
+
+  def urls_on_same_domain?(url1, url2)
+    get_domain(url1) == get_domain(url2)
+  end
+
+  def get_domain(url)
+    remove_extra_paths(url)
+  end
+
+  private
+
+  def create_absolute_url_from_base(potential_base, relative_url)
+    remove_extra_paths(potential_base) + relative_url
+  end
+
+  def remove_extra_paths(potential_base)
+    index_to_start_slash_search = potential_base.index('://')+3
+    index_of_first_relevant_slash = potential_base.index('/', index_to_start_slash_search)
+    if index_of_first_relevant_slash != nil
+      potential_base[0, index_of_first_relevant_slash]
+    else
+    	potential_base
+    end
+  end
+
+  def create_absolute_url_from_context(potential_base, relative_url)
+    absolute_url = nil
+    if potential_base.match(/\/$/)
+      absolute_url = potential_base+relative_url
+    else
+      last_index_of_slash = potential_base.rindex('/')
+      if potential_base[last_index_of_slash-2, 2] == ':/'
+        absolute_url = potential_base+'/'+relative_url
+      else
+        last_index_of_dot = potential_base.rindex('.')
+        if last_index_of_dot < last_index_of_slash
+          absolute_url = potential_base+'/'+relative_url
+        else
+          absolute_url = potential_base[0, last_index_of_slash+1] + relative_url
+        end
+      end
+    end
+    absolute_url
+  end
+end

metadata

@@ -0,0 +1,64 @@
+--- !ruby/object:Gem::Specification
+name: dk_crawler
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- Derrick
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2018-04-01 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: nokogiri
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.8'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.8'
+description: A simple web crawler gem
+email: derrickwang57@gmail.com
+executables:
+- crawler
+extensions: []
+extra_rdoc_files: []
+files:
+- bin/crawler
+- lib/command_line_argument_parser.rb
+- lib/dk_crawler.rb
+- lib/spider.rb
+- lib/url_store.rb
+- lib/url_utils.rb
+homepage: http://rubygems.org/gems/yz_crawler
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.7.3
+signing_key: 
+specification_version: 4
+summary: My first gem
+test_files: []