RubyGems - webrobots - Versions diffs - 0.0.2 → 0.0.3 - Mend

webrobots 0.0.2 → 0.0.3

Files changed (10) hide show

data/Gemfile CHANGED Viewed

@@ -3,6 +3,7 @@ source "http://rubygems.org"
 # Example:
 #   gem "activesupport", ">= 2.3.5"
 gem "racc", ">= 0"
+gem "nokogiri", ">= 1.4.4"
 # Add dependencies to develop your gem here.
 # Include everything needed to run rake, tests, features, etc.

data/Gemfile.lock CHANGED Viewed

@@ -6,6 +6,7 @@ GEM
       bundler (~> 1.0.0)
       git (>= 1.2.5)
       rake
+    nokogiri (1.4.4)
     racc (1.4.6)
     rake (0.8.7)
     rcov (0.9.9)
@@ -17,6 +18,7 @@ PLATFORMS
 DEPENDENCIES
   bundler (~> 1.0.0)
   jeweler (~> 1.5.1)
+  nokogiri (>= 1.4.4)
   racc
   rcov
   shoulda

data/README.rdoc CHANGED Viewed

@@ -2,6 +2,22 @@
 This is a library to help write robots.txt compliant web robots.
+== Usage
+  require 'webrobots'
+  require 'uri'
+  require 'net/http'
+  robots = WebRobots.new('MyBot/1.0')
+  uri = URI('http://digg.com/news/24hr')
+  if robots.disallowed?(uri)
+    STDERR.puts "Access disallowed: #{uri}"
+    exit 1
+  end
+  body = Net::HTTP.get(uri)
+  # ...
 == Contributing to webrobots
 * Check out the latest master to make sure the feature hasn't been implemented or the bug hasn't been fixed yet

data/Rakefile CHANGED Viewed

@@ -24,7 +24,7 @@ This library helps write robots.txt compliant web robots in Ruby.
   # Include your dependencies below. Runtime dependencies are required when using your gem,
   # and development dependencies are only needed for development (ie running rake tasks, tests, etc)
   #  gem.add_runtime_dependency 'jabber4r', '> 0.1'
-  #  gem.add_development_dependency 'rspec', '> 1.2.3'
+  gem.add_development_dependency 'racc'
 end
 Jeweler::RubygemsDotOrgTasks.new

data/VERSION CHANGED Viewed

	@@ -1 +1 @@
1	- 0.0.2
1	+ 0.0.3

data/lib/webrobots.rb CHANGED Viewed

@@ -1,6 +1,11 @@
 require 'webrobots/robotstxt'
 require 'uri'
 require 'net/https'
+if defined?(Nokogiri)
+  require 'webrobots/nokogiri'
+else
+  autoload :Nokogiri, 'webrobots/nokogiri'
+end
 class WebRobots
   # Creates a WebRobots object for a robot named +user_agent+, with

data/lib/webrobots/nokogiri.rb ADDED Viewed

@@ -0,0 +1,32 @@
+require 'nokogiri'
+class Nokogiri::HTML::Document
+  # Returns an array of lower-cased <meta name="ROBOTS"> tokens.  If
+  # no tag is found, returns an empty array.  An optional
+  # +custom_name+ specifies the name of a meta tag to look for ahead
+  # of "ROBOTS".  Names are compared in a case-insensitive manner.
+  def meta_robots(custom_name = nil)
+    (@meta_robots ||= {})[custom_name] =
+      (custom_name && parse_meta_robots(custom_name)) || parse_meta_robots('robots')
+  end
+  # Equivalent to meta_robots(custom_name).include?('noindex').
+  def noindex?(custom_name = nil)
+    meta_robots(custom_name).include?('noindex')
+  end
+  # Equivalent to meta_robots(custom_name).include?('nofollow').
+  def nofollow?(custom_name = nil)
+    meta_robots(custom_name).include?('nofollow')
+  end
+  private
+  def parse_meta_robots(custom_name)
+    pattern = /\A#{Regexp.quote(custom_name)}\z/i
+    meta = css('meta[@name]').find { |meta|
+      meta['name'].match(pattern)
+    } and content = meta['content'] or return []
+    content.downcase.split(/[,\s]+/)
+  end
+end

data/test/test_webrobots.rb CHANGED Viewed

@@ -283,9 +283,38 @@ Disallow: /
         assert !@testbot.allowed?("http://store.apple.com/vieworder")
         assert @msnbot.allowed?("http://store.apple.com/vieworder")
       }
-#      assert_nothing_raised {
+      assert_nothing_raised {
         assert !@testbot.allowed?("http://github.com/login")
-#      }
+      }
+    end
+  end
+  context "meta robots tag" do
+    setup do
+      @doc = Nokogiri::HTML(<<-HTML)
+<html>
+  <head>
+    <meta name="ROBOTS" content="NOFOLLOW">
+    <meta name="Slurp" content="noindex,nofollow">
+    <meta name="googlebot" content="noarchive, noindex">
+  </head>
+  <body>
+    test
+  </body>
+</html>
+      HTML
+    end
+    should "be properly parsed when given in HTML string" do
+      assert !@doc.noindex?
+      assert  @doc.nofollow?
+      assert  @doc.noindex?('slurp')
+      assert  @doc.nofollow?('slurp')
+      assert  @doc.noindex?('googlebot')
+      assert !@doc.nofollow?('googlebot')
+      assert  @doc.meta_robots('googlebot').include?('noarchive')
     end
   end
 end

data/webrobots.gemspec CHANGED Viewed

@@ -5,11 +5,11 @@
 Gem::Specification.new do |s|
   s.name = %q{webrobots}
-  s.version = "0.0.2"
+  s.version = "0.0.3"
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["Akinori MUSHA"]
-  s.date = %q{2011-01-03}
+  s.date = %q{2011-01-05}
   s.description = %q{This library helps write robots.txt compliant web robots in Ruby.
 }
   s.email = %q{knu@idaemons.org}
@@ -26,6 +26,7 @@ Gem::Specification.new do |s|
     "Rakefile",
     "VERSION",
     "lib/webrobots.rb",
+    "lib/webrobots/nokogiri.rb",
     "lib/webrobots/robotstxt.rb",
     "lib/webrobots/robotstxt.ry",
     "test/helper.rb",
@@ -46,23 +47,29 @@ Gem::Specification.new do |s|
     if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then
       s.add_runtime_dependency(%q<racc>, [">= 0"])
+      s.add_runtime_dependency(%q<nokogiri>, [">= 1.4.4"])
       s.add_development_dependency(%q<shoulda>, [">= 0"])
       s.add_development_dependency(%q<bundler>, ["~> 1.0.0"])
       s.add_development_dependency(%q<jeweler>, ["~> 1.5.1"])
       s.add_development_dependency(%q<rcov>, [">= 0"])
+      s.add_development_dependency(%q<racc>, [">= 0"])
     else
       s.add_dependency(%q<racc>, [">= 0"])
+      s.add_dependency(%q<nokogiri>, [">= 1.4.4"])
       s.add_dependency(%q<shoulda>, [">= 0"])
       s.add_dependency(%q<bundler>, ["~> 1.0.0"])
       s.add_dependency(%q<jeweler>, ["~> 1.5.1"])
       s.add_dependency(%q<rcov>, [">= 0"])
+      s.add_dependency(%q<racc>, [">= 0"])
     end
   else
     s.add_dependency(%q<racc>, [">= 0"])
+    s.add_dependency(%q<nokogiri>, [">= 1.4.4"])
     s.add_dependency(%q<shoulda>, [">= 0"])
     s.add_dependency(%q<bundler>, ["~> 1.0.0"])
     s.add_dependency(%q<jeweler>, ["~> 1.5.1"])
     s.add_dependency(%q<rcov>, [">= 0"])
+    s.add_dependency(%q<racc>, [">= 0"])
   end
 end

metadata CHANGED Viewed

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification
 name: webrobots
 version: !ruby/object:Gem::Version
-  hash: 27
+  hash: 25
   prerelease:
   segments:
   - 0
   - 0
-  - 2
-  version: 0.0.2
+  - 3
+  version: 0.0.3
 platform: ruby
 authors:
 - Akinori MUSHA
@@ -15,7 +15,7 @@ autorequire:
 bindir: bin
 cert_chain: []
-date: 2011-01-03 00:00:00 +09:00
+date: 2011-01-05 00:00:00 +09:00
 default_executable:
 dependencies:
 - !ruby/object:Gem::Dependency
@@ -33,8 +33,24 @@ dependencies:
   prerelease: false
   name: racc
 - !ruby/object:Gem::Dependency
-  type: :development
+  type: :runtime
   version_requirements: &id002 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        hash: 15
+        segments:
+        - 1
+        - 4
+        - 4
+        version: 1.4.4
+  requirement: *id002
+  prerelease: false
+  name: nokogiri
+- !ruby/object:Gem::Dependency
+  type: :development
+  version_requirements: &id003 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ">="
@@ -43,12 +59,12 @@ dependencies:
         segments:
         - 0
         version: "0"
-  requirement: *id002
+  requirement: *id003
   prerelease: false
   name: shoulda
 - !ruby/object:Gem::Dependency
   type: :development
-  version_requirements: &id003 !ruby/object:Gem::Requirement
+  version_requirements: &id004 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -59,12 +75,12 @@ dependencies:
         - 0
         - 0
         version: 1.0.0
-  requirement: *id003
+  requirement: *id004
   prerelease: false
   name: bundler
 - !ruby/object:Gem::Dependency
   type: :development
-  version_requirements: &id004 !ruby/object:Gem::Requirement
+  version_requirements: &id005 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -75,12 +91,12 @@ dependencies:
         - 5
         - 1
         version: 1.5.1
-  requirement: *id004
+  requirement: *id005
   prerelease: false
   name: jeweler
 - !ruby/object:Gem::Dependency
   type: :development
-  version_requirements: &id005 !ruby/object:Gem::Requirement
+  version_requirements: &id006 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ">="
@@ -89,9 +105,23 @@ dependencies:
         segments:
         - 0
         version: "0"
-  requirement: *id005
+  requirement: *id006
   prerelease: false
   name: rcov
+- !ruby/object:Gem::Dependency
+  type: :development
+  version_requirements: &id007 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        hash: 3
+        segments:
+        - 0
+        version: "0"
+  requirement: *id007
+  prerelease: false
+  name: racc
 description: |
   This library helps write robots.txt compliant web robots in Ruby.
@@ -112,6 +142,7 @@ files:
 - Rakefile
 - VERSION
 - lib/webrobots.rb
+- lib/webrobots/nokogiri.rb
 - lib/webrobots/robotstxt.rb
 - lib/webrobots/robotstxt.ry
 - test/helper.rb