grucrawler 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 2c1969c40c532caad82628f59ac6cd088d069af2
+  data.tar.gz: 27e7781412201b02ef9356b28f0119a2cef5d69d
+SHA512:
+  metadata.gz: 2f8704b18e02eedd2efaa39649a2e61e60bc6cfcd17a8e5e77e28e6e793d4475f5e29d887441cb0f632cd785d03e34e0c661059f810460be190ade23da2a7925
+  data.tar.gz: e10692a0971c5b427b501018fb85419d2810880225add5f2a7d7358abd4bd82b32f5ce5bc17781c582bdcde61d8429e040755d88d6cad9cacac8b7d2cc5f490c

data/.gitignore

@@ -0,0 +1,25 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp
+*.bundle
+*.so
+*.o
+*.a
+mkmf.log
+
+/.idea
+/examples

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in grucrawler.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2014 Slava Vishnyakov
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,78 @@
+# Grucrawler
+
+```ruby
+require 'grucrawler'
+
+class ItalianCrawler
+  def options
+    {
+        visit_urls_only_once: true,
+        follow_redirects: true,
+        concurrency: 5
+    }
+  end
+
+  def on_init(crawler)
+    @crawler = crawler
+  end
+
+  def on_page_received(typhoeus_response, nokogiri_html)
+    puts "GOT #{typhoeus_response.effective_url.green}"
+
+    # typhoeus_response.body
+    # typhoeus_response.request.url
+    # typhoeus_response.effective_url
+    # nokogiri_html.css('a').each |a| { puts a.text; }
+  end
+
+  def follow_link(target_url, typhoeus_response, nokogiri_html)
+    return true if target_url.include? '.it'
+
+    false
+  end
+
+  def debug(message)
+    #puts message.blue
+  end
+
+  def log_info(message)
+    puts message.yellow
+  end
+
+  def log_error(typhoeus_response, exception)
+    puts exception.to_s.red
+  end
+end
+
+c = GruCrawler.new(ItalianCrawler.new)
+# c.reset() # deletes all memory of all events - useful for restarting crawl
+c.add_url('http://www.oneworlditaliano.com/english/italian/news-in-italian.htm')
+c.run()
+```
+
+
+
+## Installation
+
+[!] Requires local Redis at the moment.
+
+Add this line to your application's Gemfile:
+
+    gem 'grucrawler'
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install grucrawler
+
+
+## Contributing
+
+1. Fork it ( https://github.com/[my-github-username]/grucrawler/fork )
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create a new Pull Request

data/Rakefile

@@ -0,0 +1,2 @@
+require "bundler/gem_tasks"
+

data/grucrawler.gemspec

@@ -0,0 +1,28 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'grucrawler/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "grucrawler"
+  spec.version       = GruCrawler::VERSION
+  spec.authors       = ["Slava Vishnyakov"]
+  spec.email         = ["bomboze@gmail.com"]
+  spec.summary       = %q{Simple crawler using Redis as backend}
+  spec.description   = %q{Simple crawler using Redis as backend}
+  spec.homepage      = "https://github.com/slava-vishnyakov/grucrawler"
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files -z`.split("\x0")
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib"]
+
+  spec.add_development_dependency "bundler", "~> 1.6"
+  spec.add_development_dependency "rake"
+
+  spec.add_dependency "typhoeus"
+  spec.add_dependency "redis"
+  spec.add_dependency "nokogiri"
+
+end

data/lib/grucrawler/queue.rb

@@ -0,0 +1,120 @@
+class GruCrawler
+  class DoNotCrawlFurther < Exception
+  end
+
+  class Queue
+    VISITED_ALREADY_KEY = 'visited_already'
+    DOMAIN_VISITS_KEY = 'domain_visits'
+    QUEUE_KEY = 'queue'
+
+    def initialize(namespace, visit_once)
+      @redis = Redis.new
+      @rns = namespace + ':'
+      @concurrent_requests = 0
+      @tmp_block = {}
+      @domains_throttle = Hash.new(0.0)
+      @visit_once = visit_once
+    end
+
+    def reset
+      @redis.del(@rns + DOMAIN_VISITS_KEY)
+      @redis.del(@rns + QUEUE_KEY)
+      @redis.del(@rns + VISITED_ALREADY_KEY)
+    end
+
+    def next_url
+      url = ''
+
+      100.times do
+        url = random_url_from_queue()
+
+        if visited_already(url) or not can_visit_now(url)
+          url = nil
+          next
+        end
+
+        break
+      end
+
+      @tmp_block[url] = true
+
+      url
+    end
+
+    MIN_TIME_TO_WAIT = 20
+
+    def can_visit_now(url)
+      return false if @tmp_block[url]
+
+      last_visit = last_visit_to_domain(url)
+      time_passed = Time.now.to_f - last_visit
+
+      time_passed > MIN_TIME_TO_WAIT
+    end
+
+    def started(url)
+      set_last_visit_to_domain(url)
+
+      @concurrent_requests += 1
+    end
+
+    def finished(url)
+      @tmp_block.delete(url)
+      set_visited_already(url)
+      remove_url_from_queue(url) if url
+      @concurrent_requests -= 1
+    end
+
+    def count
+      @concurrent_requests
+    end
+
+    def set_last_visit_to_domain(url)
+      time = Time.now.to_f
+      @redis.hset(@rns + DOMAIN_VISITS_KEY, domain(url), time)
+    end
+
+    def last_visit_to_domain(url)
+      @redis.hget(@rns + DOMAIN_VISITS_KEY, domain(url)).to_f
+    end
+
+
+    def remove_url_from_queue(url)
+      @redis.srem(@rns + QUEUE_KEY, url)
+    end
+
+    def random_url_from_queue
+      @redis.srandmember(@rns + QUEUE_KEY)
+    end
+
+    def push(url)
+      @redis.sadd(@rns + QUEUE_KEY, url) == 1
+    end
+
+
+    def visited_already(url)
+      return false unless @visit_once
+      @redis.sismember(@rns + VISITED_ALREADY_KEY, url)
+    end
+
+    def set_visited_already(url)
+      return unless @visit_once
+      @redis.sadd(@rns + VISITED_ALREADY_KEY, url)
+    end
+
+
+    # TODO: PublicSuffix
+    def domain(url)
+      begin
+        uri = URI.parse(url)
+      rescue URI::InvalidURIError
+        return nil
+      end
+
+      return nil if uri.host.nil?
+      host = uri.host.downcase
+      host = host.start_with?('www.') ? host[4..-1] : host
+      host.match(/\w+\.\w+$/)[0]
+    end
+  end
+end

data/lib/grucrawler/version.rb

@@ -0,0 +1,3 @@
+class GruCrawler
+  VERSION = "0.0.1"
+end

data/lib/grucrawler.rb

@@ -0,0 +1,94 @@
+require "grucrawler/version"
+require "grucrawler/queue"
+require "typhoeus"
+require "redis"
+require "uri"
+require "nokogiri"
+
+class GruCrawler
+  class DoNotCrawlFurther < Exception
+  end
+
+  def initialize(rules)
+    @crawler = rules
+    @options = @crawler.options()
+    @queue = GruCrawler::Queue.new(@crawler.class.name, @options[:visit_urls_only_once])
+
+    @crawler.on_init(self)
+  end
+
+  def run
+    @hydra = Typhoeus::Hydra.new()
+    @concurrency = @options[:concurrency] || 5
+    crawl_more()
+    @hydra.run
+  end
+
+  def add_url(url)
+    @queue.push(url)
+  end
+
+  def add_from_queue
+    url = @queue.next_url()
+    return false unless url
+
+    request = Typhoeus::Request.new(url, followlocation: @options[:follow_redirects], accept_encoding: 'gzip')
+    @queue.started(url)
+
+    request.on_complete do |response|
+      on_response(response)
+    end
+
+    @crawler.debug("#{Time.now} started URL #{url}")
+    @hydra.queue(request)
+
+    true
+  end
+
+  def reset
+    @queue.reset
+  end
+
+  def on_response(response)
+    @crawler.debug("#{Time.now} ended URL #{response.request.url}")
+    @queue.finished(response.request.url)
+
+    crawl_more()
+
+    nokogiri = Nokogiri::HTML(response.body)
+
+    begin
+      @crawler.on_page_received(response, nokogiri)
+    rescue
+      @crawler.log_error(response, $!)
+    end
+
+    queue_links(response, nokogiri)
+
+    crawl_more()
+  end
+
+  def crawl_more
+    while @queue.count < @concurrency
+      break unless add_from_queue()
+    end
+  end
+
+  def queue_links(response, nokogiri)
+    nokogiri.css('a').each do |link|
+      next unless link['href']
+
+      begin
+        url = URI.join(response.effective_url, link['href']).to_s
+      rescue
+        next
+      end
+      if @crawler.follow_link(url, response, nokogiri)
+        added = add_url(url)
+        @crawler.debug("#{Time.now} queued #{url}") if added
+      end
+    end
+  end
+
+
+end

metadata

@@ -0,0 +1,123 @@
+--- !ruby/object:Gem::Specification
+name: grucrawler
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- Slava Vishnyakov
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2014-11-18 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.6'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.6'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: typhoeus
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: redis
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: nokogiri
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Simple crawler using Redis as backend
+email:
+- bomboze@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- grucrawler.gemspec
+- lib/grucrawler.rb
+- lib/grucrawler/queue.rb
+- lib/grucrawler/version.rb
+homepage: https://github.com/slava-vishnyakov/grucrawler
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.2.2
+signing_key: 
+specification_version: 4
+summary: Simple crawler using Redis as backend
+test_files: []