rubygems-crawler 0.0.1

data/bin/rubygems-gems-crawler

@@ -0,0 +1,13 @@
+#!/usr/bin/env ruby
+
+# Define an env variable MONGO_URI like:
+# MONGO_URI='mongodb://localhost:27017/'
+
+require 'rubygems-crawler/gems_crawler'
+
+mongo_uri = ENV['MONGO_URI'] || 'mongodb://localhost:27017/'
+mongo_client = Mongo::MongoClient.from_uri(mongo_uri)
+mongo_db = mongo_client.db('rubygems')
+
+g = RubyGems::GemsCrawler.new mongo_db
+g.crawl_from 'a'

data/bin/rubygems-web-crawler

@@ -0,0 +1,13 @@
+#!/usr/bin/env ruby
+
+# Define an env variable MONGO_URI like:
+# MONGO_URI='mongodb://localhost:27017/'
+
+require 'rubygems-crawler/web_crawler'
+
+mongo_uri = ENV['MONGO_URI'] || 'mongodb://localhost:27017/'
+mongo_client = Mongo::MongoClient.from_uri(mongo_uri)
+mongo_db = mongo_client.db('rubygems')
+
+w = RubyGems::WebCrawler.new mongo_db
+('A'..'Z').each {|seed| w.crawl(seed) } 

data/lib/rubygems-crawler/gems_crawler.rb

@@ -0,0 +1,42 @@
+require 'gems'
+require 'mongo'
+
+# A very simple gems crawler for Rubygems.org
+module RubyGems
+  class GemsCrawler
+    
+    GRACE_PERIOD = 5  # be gentle
+    
+    def initialize(mongo)
+      @mongo = mongo
+    end
+    
+    def crawl_from(initial_name='a')
+      #name: {'$gte' => initial_name} - to filter by name
+      @mongo[:gems].find({owners: nil}, {fields: ["name"]}).each_slice(10) do |bulk|
+        bulk.each do |mongo_doc|
+          crawl(mongo_doc['name'])
+          sleep GRACE_PERIOD  #be nice
+        end
+      end
+    end
+    
+    def crawl(gem_name)
+      STDOUT.puts "[RubyGems Web Crawler] Acquiring data for gem #{gem_name}"
+      
+      gem_object = Gems.info(gem_name)
+      gem_object['versions'] = Gems.versions(gem_name)
+      gem_object['owners'] = Gems.owners(gem_name)
+      
+      save(gem_object)
+    rescue
+      STDERR.puts "[RubyGems Web Crawler] Error while acquiring data for gem #{gem_name}"
+    end
+    
+    # Save all the gem data into Mongo
+    def save(gem_object)
+      @mongo[:gems].find_and_modify(query: {name: gem_object['name']}, update: gem_object)
+    end
+    
+  end
+end

data/lib/rubygems-crawler/web_crawler.rb

@@ -0,0 +1,90 @@
+require 'net/http'
+require 'nokogiri'
+require 'mongo'
+
+# A very simple web crawler for Rubygems.org
+module RubyGems
+  class WebCrawler
+    
+    BASE_URL = 'http://rubygems.org'
+    REQUEST_HEADERS = {'User-Agent'=>'rubygems-crawler'}
+    TIMEOUT=30
+    GRACE_PERIOD=1  #Sleep for a while - be gentle
+    
+    def initialize(mongo)
+      @mongo = mongo
+    end
+    
+    # Crawl all the pages of RubyGems, given an initial letter and save the data into MongoDB
+    def crawl(letter='A')
+      url = "#{BASE_URL}/gems?letter=#{letter}"
+      while url && gems = download_page(url)
+        save_gems(gems[:gems])
+        STDOUT.puts "[RubyGems Web Crawler] [#{url}] - Acquired #{gems[:gems].count} gems"
+        
+        url = (gems[:next_path]) ? "#{BASE_URL}#{gems[:next_path]}" : nil
+        sleep GRACE_PERIOD
+      end
+    end
+    
+    # Download an HTML page given an url, parse the HTML and convert the result back into an HASH
+    def download_page(url)
+      STDOUT.puts "Acquiring #{url}"
+      
+      network_res = network_call(url, REQUEST_HEADERS, TIMEOUT)
+      return parse_content(network_res[:response]) if network_res && network_res[:response]
+    end
+    
+    #Execute a GET HTTP call to url given the specified headers
+    def network_call(url, request_headers={}, timeout = nil)
+
+      retries = 0
+      begin
+        uri = URI.parse(url.ascii_only? ? url : URI.escape(url))
+        http = Net::HTTP.new(uri.host, uri.port)
+
+        unless timeout.nil?
+          http.open_timeout = timeout
+          http.read_timeout = timeout
+        end
+
+        request = Net::HTTP::Get.new(uri.request_uri, request_headers)
+        response = http.request(request)
+
+      rescue Timeout::Error, Net::HTTPBadResponse, EOFError => e
+        retries += 1
+        retry unless retries > 3
+        return {error: e, code: 0}
+      end
+
+      result = {:code=>response.code.to_i}    
+      result[:response] = response.body if response.code.to_s == '200'
+      result
+    end
+    
+    # Parse the HTML of the page extracting gem names and total number of pages
+    def parse_content(response)
+      gem_res = {:gems => [], :next_path => nil}
+      
+      html_doc = Nokogiri::HTML(response)
+      
+      html_doc.css('.gems li a>strong').each do |node|
+        node.content =~ /(.+)\s\(.+\)/
+        gem_res[:gems] << $1
+      end      
+  
+      next_page = html_doc.css('.next_page').first
+      if next_page
+        gem_res[:next_path] = next_page.attr('href')
+      end
+      
+      gem_res
+    end
+    
+    # Save all the gem names into Mongo
+    def save_gems(gems)
+      gems.each {|gem_name| @mongo[:gems].insert({name: gem_name}) }
+    end
+    
+  end
+end

metadata

@@ -0,0 +1,138 @@
+--- !ruby/object:Gem::Specification
+name: rubygems-crawler
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+  prerelease: 
+platform: ruby
+authors:
+- Luca Bonmassar
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2013-10-28 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: nokogiri
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 1.5.5
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 1.5.5
+- !ruby/object:Gem::Dependency
+  name: mongo
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 1.8.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 1.8.0
+- !ruby/object:Gem::Dependency
+  name: bson_ext
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 1.8.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 1.8.0
+- !ruby/object:Gem::Dependency
+  name: gems
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.8.3
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.8.3
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: A very simple crawler for RubyGems.org used to demo the power of ElasticSearch
+  at RubyConf 2013
+email:
+- luca@gild.com
+executables:
+- rubygems-gems-crawler
+- rubygems-web-crawler
+extensions: []
+extra_rdoc_files: []
+files:
+- lib/rubygems-crawler/gems_crawler.rb
+- lib/rubygems-crawler/web_crawler.rb
+- bin/rubygems-gems-crawler
+- bin/rubygems-web-crawler
+homepage: http://www.gild.com
+licenses: []
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+      segments:
+      - 0
+      hash: -4028937865614676331
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+      segments:
+      - 0
+      hash: -4028937865614676331
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.24
+signing_key: 
+specification_version: 3
+summary: A very simple crawler for RubyGems.org
+test_files: []