zy_crawler 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 2024898f0c89209ea0a427d2fa0c03a1c628f486baac028522c9c2bb400feab1
+  data.tar.gz: f1e0d8bb0b62406e197cec47fd2e5c200a35669cca578b3f64bb788bd270b492
+SHA512:
+  metadata.gz: a4ee958478df918e2ddf8b7fe6bb12949e973338aceb6ea0aa9024ef8c2404c0ea0c64bc69b7b6c2cc01d7d1e7d1476539430dde715025899f7a70218b2b01db
+  data.tar.gz: 10fcbdf784221de565506976d6aa9359203ef32b79efe074b745abaf420df96cff24e55e114795fcf05337e557127cb59cf6f08db0e9351782dcc31a93b6e2ca

data/bin/zycrawler

@@ -0,0 +1,5 @@
+#!/usr/bin/env ruby
+require 'zy_crawler'
+
+crawler = ZyCrawler.new
+crawler.crawl

data/lib/command_line_argument_parser.rb

@@ -0,0 +1,55 @@
+require 'getoptlong'
+# command line argument parser
+class CommandLineArgumentParser
+  WEB_CRAWLER = 'web'.freeze
+  DOMAIN_CRAWLER = 'domain'.freeze
+  attr_reader :crawl_type, :crawl_depth, :page_limit, :url_file
+
+  def initialize
+    unless ARGV.length >= 1
+      display_usage
+      exit
+    end
+    @opts = GetoptLong.new(
+      ['--crawl', '-c', GetoptLong::REQUIRED_ARGUMENT],
+      ['--crawl-depth', '-d', GetoptLong::OPTIONAL_ARGUMENT],
+      ['--page-limit', '-p', GetoptLong::OPTIONAL_ARGUMENT],
+      ['--url-file', '-f', GetoptLong::OPTIONAL_ARGUMENT]
+    )
+    @crawl_type = 'data.txt'
+    @crawl_depth = 3
+    @page_limit = 100
+    @url_file = 'urls.txt'
+  end
+
+  def display_usage
+    p 'Sample usage:'
+    p "ruby zy_crawler.rb -c web -d 3 -p 100 -f 'urls.txt'"
+    p "-c must be either 'web' or 'domain', will default to 'web' if you type garbage"
+  end
+
+  def parse_arguments
+    @opts.each do |opt, arg|
+      case opt
+      when '--crawl'
+        ensure_crawl_type_correct(arg)
+      when '--crawl-depth'
+        @crawl_depth = arg.to_i
+      when '--page-limit'
+        @page_limit = arg.to_i
+      when '--url-file'
+        @url_file = arg
+      else
+        puts 'what happend?'
+      end
+    end
+  end
+
+  def ensure_crawl_type_correct(value)
+    if value != WEB_CRAWLER && value != DOMAIN_CRAWLER
+      @crawl_type = WEB_CRAWLER
+    else
+      @crawl_type = value
+    end
+  end
+end

data/lib/spider.rb

@@ -0,0 +1,84 @@
+require 'net/http'
+require 'uri'
+require 'open-uri'
+require 'rubygems'
+require 'hpricot'
+require 'url_utils'
+# Young Spider
+class Spider
+  include UrlUtils
+
+  def initialize
+    @already_visited = {}
+  end
+
+  def crawl_web(urls, depth = 2, page_limit = 100)
+    depth.times do
+      next_urls = []
+      urls.each do |url|
+        url_object = open_url(url)
+        next if url_object.nil?
+
+        url = upate_url_if_redirected(url_object)
+        parsed_doc = parse_url(url_object)
+        next if parsed_doc.nil?
+
+        @already_visited[url] == true if @already_visited[url].nil?
+        return if @already_visited.size == page_limit
+
+        next_urls += (find_urls_on_page(parsed_doc, url) - @already_visited.keys)
+        next_urls.uniq!
+      end
+      urls = next_urls
+    end
+  end
+
+  def crawl_domain(url, page_limit = 100)
+    return if @already_visited.size == page_limit
+
+    url_object = open_url(url)
+    return if url_object.nil?
+
+    parsed_doc = parse_url(url_object)
+    return if parsed_doc.nil?
+
+    @already_visited[url] == true if @already_visited[url].nil?
+    page_urls = find_urls_on_page(parsed_doc, url)
+    page_urls.each do |page_url|
+      if urls_on_same_domain?(url, page_url) && @already_visited[page_url].nil?
+        crawl_domain(page_url)
+      end
+    end
+  end
+
+  private
+
+  def open_url(url)
+    URI.open(url)
+  rescue
+    puts 'Unable to open url: ' + url
+  end
+
+  def upate_url_if_redirected(url_object)
+    url_object.base_uri.to_s
+  end
+
+  def parse_url(url_object)
+    doc = Hpricot(url_object) #nokogiri
+    puts 'Crawling url ' + url_object.base_uri.to_s
+    doc
+  rescue
+    puts 'Could not parse url: ' + url_object.base_uri.to_s
+  end
+
+  def find_urls_on_page(pared_doc, current_url)
+    pared_doc.search('a[@href]').each_with_object([]) do |x, urls_list|
+      new_url = x['href'].split('#')[0]
+      if new_url
+        # complicated feature: make_absolute
+        new_url = make_absolute(current_url, new_url) if relative?(new_url)
+        urls_list.push(new_url)
+      end
+    end
+  end
+end

data/lib/url_store.rb

@@ -0,0 +1,26 @@
+# Memory URL Store
+class URLStore
+  attr_reader :urls
+  alias get_urls urls
+
+  def initialize(url_file)
+    @urls = read_urls_from_file(url_file)
+  end
+
+  def firt_url
+    @urls[0]
+  end
+
+  private
+
+  def read_urls_from_file(url_file)
+    urls = []
+    File.open(url_file, 'r') do |file|
+      file.readlines.each do |line|
+        urls.push(line.chomp)
+      end
+    end
+    urls
+  end
+
+end

data/lib/url_utils.rb

@@ -0,0 +1,56 @@
+# URL Tools
+module UrlUtils
+  def relative?(url)
+    !url.match(/^http/)
+  end
+
+  def make_absolute(potential_base, relative_url)
+    if relative_url =~ /^\//
+      create_absolute_url_from_base(potential_base, relative_url)
+    else
+      create_abs_url_from_ctx(potential_base, relative_url)
+    end
+  end
+
+  def urls_on_same_domain?(url1, url2)
+    get_domain(url1) == get_domain(url2)
+  end
+
+  def get_domain(url)
+    remove_extra_paths(url)
+  end
+
+  def create_absolute_url_from_base(potential_base, relative_url)
+    remove_extra_paths(potential_base) + relative_url
+  end
+
+  def remove_extra_paths(potential_base)
+    index_to_start_slash_search = potential_base.index('://') + 3
+    index_of_first_relevant_slash = potential_base.index('/', index_to_start_slash_search)
+    if !index_of_first_relevant_slash.nil?
+      potential_base[0, index_of_first_relevant_slash]
+    else
+      potential_base
+    end
+  end
+
+  def create_abs_url_from_ctx(potential_base, relative_url)
+    absolute_url = nil
+    if potential_base =~ /\/$/
+      absolute_url = potential_base + relative_url
+    else
+      last_index_of_slash = potential_base.rindex('/')
+      if potential_base[last_index_of_slash - 2, 2] == ':/'
+        absolute_url = potential_base + '/' + relative_url
+      else
+        last_index_of_dot = potential_base.rindex('.')
+        if last_index_of_dot < last_index_of_slash
+          absolute_url = potential_base + '/' + relative_url
+        else
+          absolute_url = potential_base[0, last_index_of_slash + 1] + relative_url
+        end
+      end
+    end
+    absolute_url
+  end
+end

data/lib/zy_crawler.rb

@@ -0,0 +1,23 @@
+require 'command_line_argument_parser'
+require 'spider'
+require 'url_store'
+
+# ZyCrawler is from Jason zhao's YzCrawler
+class ZyCrawler
+  def initialize
+    @argument_parser = CommandLineArgumentParser.new
+    @argument_parser.parse_arguments
+    @spider = Spider.new
+    @url_store = URLStore.new(@argument_parser.url_file)
+  end
+
+  def crawl
+    if @argument_parser.crawl_type == CommandLineArgumentParser::WEB_CRAWLER
+      @spider.crawl_web(@url_store.get_urls,
+                        @argument_parser.crawl_depth,
+                        @argument_parser.page_limit)
+    else
+      @spider.crawl_domain(@url_store.firt_url, @argument_parser.page_limit)
+    end
+  end
+end

metadata

@@ -0,0 +1,66 @@
+--- !ruby/object:Gem::Specification
+name: zy_crawler
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- uuen sky
+autorequire:
+bindir: bin
+cert_chain: []
+date: 2022-03-08 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: hpricot
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.8'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.8'
+description: A simple crawler demo crawler
+email: uuensky@163.com
+executables:
+- zycrawler
+extensions: []
+extra_rdoc_files: []
+files:
+- bin/zycrawler
+- lib/command_line_argument_parser.rb
+- lib/spider.rb
+- lib/url_store.rb
+- lib/url_utils.rb
+- lib/zy_crawler.rb
+homepage: https://rubygems.org/gems/zycrawler
+licenses:
+- MIT
+metadata:
+  changelog_uri: https://github.com/uuensky/zycrawler/blob/master/CHANGELOG.md
+  homepage_uri: https://rubygems.org/gems/zycrawler
+  source_code_uri: https://github.com/uuensky/zycrawler.git
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.2.32
+signing_key:
+specification_version: 4
+summary: A yong spider
+test_files: []