ruby-crawler 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 939a77945341f3b298f7ee95d448e49d423ace18
+  data.tar.gz: 19b95592ff7960a39fee570adfd1983c89d12da4
+SHA512:
+  metadata.gz: 9d411c506bced66661da541bfe1a4e6ec10903a9800789481b8788f7cad98f2de8e9572327862aa381fa6d02ef902533df5ac6c4bcf421a002344a11eae5d305
+  data.tar.gz: e09ecb14da9a0c246cd825139366dad39f8497b23fd9aeee264c28e05560a4743ec8d3f1cf351686542a350f8a8cfd942747982006cf01ae21a658dd09d0d357

data/.gitignore

@@ -0,0 +1,19 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp
+coverage
+dump.rdb

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in crawler.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2014 Adam Ryan
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,39 @@
+# Crawler
+
+Crawler is a straightforward gem that will index a given domain, and store the results in redis.
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+    gem 'ruby-crawler'
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install ruby-crawler
+
+## Usage
+
+Crawl a given domain
+```ruby
+crawler = Crawler.new('http://domain.com')
+crawler.crawl
+```
+
+
+View Results
+```ruby
+crawler.results
+```
+
+## Contributing
+
+1. Fork it ( http://github.com/<my-github-username>/crawler/fork )
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create new Pull Request

data/Rakefile

@@ -0,0 +1,11 @@
+require 'bundler'
+require 'bundler/gem_tasks'
+Bundler::GemHelper.install_tasks
+
+require 'rake/testtask'
+Rake::TestTask.new(:test) do |t|
+  t.libs << 'test'
+  t.pattern = 'test/**/*_test.rb'
+end
+
+task :default => :test

data/crawler.gemspec

@@ -0,0 +1,30 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'crawler/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "ruby-crawler"
+  spec.version       = Crawler::VERSION
+  spec.authors       = ["Adam Ryan"]
+  spec.email         = ["adam.g.ryan@gmail.com"]
+  spec.summary       = %q{Simple ruby web crawler}
+  spec.description   = %q{Simple ruby web crawler}
+  spec.homepage      = ""
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files`.split($/)
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib"]
+
+  spec.add_dependency "nokogiri"
+  spec.add_dependency "addressable"
+  spec.add_dependency "redis"
+
+  spec.add_development_dependency "bundler", "~> 1.5"
+  spec.add_development_dependency "rake"
+  spec.add_development_dependency "minitest"
+  spec.add_development_dependency "webmock"
+  spec.add_development_dependency "simplecov"
+end

data/lib/crawler.rb

@@ -0,0 +1,59 @@
+require 'addressable/uri'
+require 'crawler/document'
+require 'crawler/error'
+require 'crawler/formatting'
+require 'crawler/index'
+require 'crawler/validations'
+require 'crawler/version'
+
+module Crawler
+  class << self
+    include Formatting
+    include Validations
+
+    attr_accessor :base_uri, :base_url, :index
+
+    # Initialize a new Crawler
+    #
+    # base_uri => base_uri to crawl
+    # base_url => base_url to crawl
+    #
+    def new(domain)
+      @base_uri = Addressable::URI.parse(domain.strip)
+      validate_protocol
+
+      @base_url = construct_url base_uri
+      self
+    end
+
+    # Performs crawl of domain, indexes results
+    #
+    def crawl(url = base_url)
+      document = Crawler::Document.new(url)
+      index.consume_document url.sub(base_url, ''), document
+
+      paths_queue = index.get_paths_to_visit
+      next_path = paths_queue[0]
+
+      print " Pages remaing - #{paths_queue.count}    \r"
+      crawl "#{base_url}#{next_path}" if next_path
+    end
+
+    # Returns the indexed results of a crawl
+    #
+    def results
+      index.results
+    end
+
+    private
+
+    # Crawler::Index
+    # New Index used to record results of domain crawl
+    #
+    def index
+      @index ||= Crawler::Index.new(base_uri)
+    end
+
+  end
+
+end

data/lib/crawler/document.rb

@@ -0,0 +1,33 @@
+require 'crawler/document_parser'
+require 'crawler/formatting'
+
+module Crawler
+  class Document
+    include DocumentParser
+    include Formatting
+
+    attr_accessor :uri, :url, :content, :links, :domain_specific_paths, :static_assets
+
+    # Intialize a new Document
+    #
+    # uri => uri of the document
+    # url => url of the document
+    # links => links found in the document
+    # domain_specific_paths => paths in the document related to the crawler's base domain
+    # static_assets => static_assets found in the document
+    #
+    def initialize(url)
+      @uri = Addressable::URI.parse(url.strip)
+      @url = construct_url uri
+      @links = extract_links
+      @domain_specific_paths = extract_domain_specific_paths
+      @static_assets = extract_assets
+    end
+
+    # Nokogiri::HTML::Document
+    #
+    def content
+      @content ||= parse_content uri
+    end
+  end
+end

data/lib/crawler/document_parser.rb

@@ -0,0 +1,45 @@
+require 'nokogiri'
+require 'crawler/http'
+require 'crawler/formatting'
+
+module Crawler
+  module DocumentParser
+    include Formatting
+    include Http
+
+    private
+
+    # Parses the HTML from an http response
+    #
+    def parse_content(uri)
+      Nokogiri::HTML request(uri)
+    end
+
+    # Returns the links from the html document
+    #
+    def extract_links
+      content.css('a').map { |a| a['href'] unless a['href'] == '#' }.compact.uniq
+    end
+
+    # Returns the static assets from the html document
+    #
+    def extract_assets
+      assets = content.css('img', 'script').map { |i| i['src'] }
+      assets |= content.css('video').map { |v| v['poster'] }
+      assets |= content.css('link').map { |l| l['href'] }
+
+      assets.compact.uniq
+    end
+
+    # Returns the paths that are related to the given domain
+    #
+    def extract_domain_specific_paths
+      links.map do |link|
+        uri = Addressable::URI.parse(link.strip)
+        if uri.hostname.nil? || uri.hostname == @uri.hostname
+          normalize_path uri.path
+        end
+      end.compact
+    end
+  end
+end

data/lib/crawler/error.rb

@@ -0,0 +1,10 @@
+module Crawler
+
+  # Base Crawler Error
+  #
+  class Error < StandardError; end
+
+  # Raised when invalid protocol is used
+  #
+  class InvalidProtocolError < Error; end
+end

data/lib/crawler/formatting.rb

@@ -0,0 +1,16 @@
+module Crawler
+  module Formatting
+
+    # normalize paths => '/path/to/'
+    #
+    def normalize_path(path)
+      "#{path}/".gsub(/\/+/, '/')
+    end
+
+    # constructs a full url from a given uri object
+    #
+    def construct_url(uri)
+      "#{uri.scheme}://#{uri.host}#{uri.path}"
+    end
+  end
+end

data/lib/crawler/http.rb

@@ -0,0 +1,17 @@
+require 'net/http'
+require 'open-uri'
+
+module Crawler
+  module Http
+
+    # Open URI HTTP request
+    #
+    def request(uri)
+      begin
+        open(uri)
+      rescue
+        ''
+      end
+    end
+  end
+end

data/lib/crawler/index.rb

@@ -0,0 +1,63 @@
+require 'crawler/formatting'
+require 'crawler/storage'
+
+module Crawler
+  class Index
+    include Formatting
+    include Storage
+
+    attr_accessor :base_uri
+
+    # New Index to record paths for a given domain
+    #
+    def initialize(base_uri)
+      @base_uri = base_uri
+      clear_stored_results
+    end
+
+    # Ingests a Crawler::Document, stores all relevant data in redis
+    # Updates pages that need to be visited as well as pages that have been visited already
+    #
+    def consume_document(path, document)
+      path = normalize_path path
+      new_links = document.domain_specific_paths.map { |path| normalize_path path }
+
+      store_path path
+
+      store_path_visited path
+
+      store_path_assets path, document.static_assets
+
+      store_path_links_to path, new_links
+
+      store_paths_to_visit(new_links - get_paths_visited)
+
+      remove_path_from_queue path
+
+      update_paths_linked_to_from_path(document)
+    end
+
+    # Returns the data associated with an indexed domain
+    #
+    def results
+      get_domain_data
+    end
+
+    private
+
+    # Records incoming links for pages
+    # Uses the current path as the incoming link
+    # Records the current_path as incoming on all links found in the current document
+    #
+    def update_paths_linked_to_from_path(document)
+      document.domain_specific_paths.each do |url|
+        link_uri_path = normalize_path Addressable::URI.parse(url.strip).path
+        document_uri_path = normalize_path document.uri.path
+        next if link_uri_path == document_uri_path
+
+        store_path link_uri_path
+        store_path_linked_to_from(link_uri_path, [document_uri_path])
+      end
+    end
+  end
+end

data/lib/crawler/storage.rb

@@ -0,0 +1,166 @@
+require 'redis'
+
+module Crawler
+  module Storage
+
+    # Stores a path for the domain
+    #
+    def store_path(path)
+      redis.sadd stored_paths_key, path
+    end
+
+    # Stores the static assets for a path
+    #
+    def store_path_assets(path, assets)
+      return if assets.empty?
+      redis.sadd path_assets_key(path), assets
+    end
+
+    # Stores paths that the current path links to
+    #
+    def store_path_links_to(path, links)
+      return if links.empty?
+      redis.sadd path_links_to_key(path), links
+    end
+
+    # Stores paths that link to the current path
+    #
+    def store_path_linked_to_from(path, links)
+      return if links.empty?
+      redis.sadd path_linked_to_from_key(path), links
+    end
+
+    # Stores paths that have been indexed for a domain
+    #
+    def store_path_visited(path)
+      redis.sadd paths_visited_key, path
+    end
+
+    # Stores paths that need to be visited for a domain
+    #
+    def store_paths_to_visit(paths)
+      return if paths.empty?
+      redis.sadd paths_to_visit_key, paths
+    end
+
+    # Removes a path from paths that need to be visited
+    #
+    def remove_path_from_queue(path)
+      redis.srem paths_to_visit_key, path
+    end
+
+    # Returns known paths for domain
+    #
+    def get_stored_paths
+      redis.smembers stored_paths_key
+    end
+
+    # Returns paths that haven't been indexed
+    #
+    def get_paths_to_visit
+      redis.smembers paths_to_visit_key
+    end
+
+    # Returns assets for a path
+    #
+    def get_path_assets(path)
+      redis.smembers path_assets_key(path)
+    end
+
+    # Returns links that a path links to
+    #
+    def get_path_links_to(path)
+      redis.smembers path_links_to_key(path)
+    end
+
+    # Returns links that link to a path
+    #
+    def get_path_linked_to_from(path)
+      redis.smembers path_linked_to_from_key(path)
+    end
+
+    # Returns paths that have been indexed for a domain
+    #
+    def get_paths_visited
+      redis.smembers paths_visited_key
+    end
+
+    # Returns paths and associated data for a domain
+    #
+    def get_domain_data
+      get_stored_paths.inject({ 'domain' => base_uri.hostname, 'paths' => {}}) do |hsh, path|
+        hsh['paths'][path] = get_path_data(path)
+        hsh
+      end
+    end
+
+    # Returns assets, links to, and links for a given path
+    #
+    def get_path_data(path)
+      {
+        'asset_dependencies' => get_path_assets(path),
+        'links_to' => get_path_links_to(path),
+        'linked_to_from' => get_path_linked_to_from(path)
+      }
+    end
+
+    # Deletes all data for a domain
+    #
+    def clear_stored_results
+      paths = get_stored_paths
+      redis.pipelined do
+        paths.each do |path|
+          [stored_paths_key, paths_visited_key, paths_to_visit_key].each do |key|
+            redis.del key
+          end
+
+          clear_path_results(path)
+        end
+      end
+    end
+
+    # Deletes all path data for a path
+    #
+    def clear_path_results(path)
+      [path_assets_key(path), path_links_to_key(path), path_linked_to_from_key(path)].each do |key|
+        redis.del key
+      end
+    end
+
+    # Redis
+    #
+    def redis
+      @redis ||= Redis.new
+    end
+
+    private
+
+    def path_assets_key(path)
+      "#{prefix}-path-assets-#{base_uri.hostname}-#{path}"
+    end
+
+    def path_links_to_key(path)
+      "#{prefix}-path-links-to-#{base_uri.hostname}-#{path}"
+    end
+
+    def path_linked_to_from_key(path)
+      "#{prefix}-path-linked-to-from-#{base_uri.hostname}-#{path}"
+    end
+
+    def stored_paths_key
+      "#{prefix}-paths-#{base_uri.hostname}"
+    end
+
+    def paths_visited_key
+      "#{prefix}-paths-visited-#{base_uri.hostname}"
+    end
+
+    def paths_to_visit_key
+      "#{prefix}-queued-paths-#{base_uri.hostname}"
+    end
+
+    def prefix
+      'crawler'
+    end
+  end
+end