RubyGems - webrobots - Versions diffs - 0.0.2 → 0.0.3 - Mend

webrobots 0.0.2 → 0.0.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (10) hide show

data/Gemfile CHANGED Viewed

@@ -3,6 +3,7 @@ source "http://rubygems.org"
 # Example:
 #   gem "activesupport", ">= 2.3.5"
 gem "racc", ">= 0"
+gem "nokogiri", ">= 1.4.4"
 # Add dependencies to develop your gem here.
 # Include everything needed to run rake, tests, features, etc.

data/Gemfile.lock CHANGED Viewed

@@ -6,6 +6,7 @@ GEM
       bundler (~> 1.0.0)
       git (>= 1.2.5)
       rake
+    nokogiri (1.4.4)
     racc (1.4.6)
     rake (0.8.7)
     rcov (0.9.9)
@@ -17,6 +18,7 @@ PLATFORMS
 DEPENDENCIES
   bundler (~> 1.0.0)
   jeweler (~> 1.5.1)
+  nokogiri (>= 1.4.4)
   racc
   rcov
   shoulda

data/README.rdoc CHANGED Viewed

@@ -2,6 +2,22 @@
 This is a library to help write robots.txt compliant web robots.
+== Usage
+  require 'webrobots'
+  require 'uri'
+  require 'net/http'
+  robots = WebRobots.new('MyBot/1.0')
+  uri = URI('http://digg.com/news/24hr')
+  if robots.disallowed?(uri)
+    STDERR.puts "Access disallowed: #{uri}"
+    exit 1
+  end
+  body = Net::HTTP.get(uri)
+  # ...
 == Contributing to webrobots
 * Check out the latest master to make sure the feature hasn't been implemented or the bug hasn't been fixed yet

data/Rakefile CHANGED Viewed

@@ -24,7 +24,7 @@ This library helps write robots.txt compliant web robots in Ruby.
   # Include your dependencies below. Runtime dependencies are required when using your gem,
   # and development dependencies are only needed for development (ie running rake tasks, tests, etc)
   #  gem.add_runtime_dependency 'jabber4r', '> 0.1'
-  #  gem.add_development_dependency 'rspec', '> 1.2.3'
+  gem.add_development_dependency 'racc'
 end
 Jeweler::RubygemsDotOrgTasks.new

data/VERSION CHANGED Viewed

	@@ -1 +1 @@
1	- 0.0.2
1	+ 0.0.3

data/lib/webrobots.rb CHANGED Viewed

@@ -1,6 +1,11 @@
 require 'webrobots/robotstxt'
 require 'uri'
 require 'net/https'
+if defined?(Nokogiri)
+  require 'webrobots/nokogiri'
+else
+  autoload :Nokogiri, 'webrobots/nokogiri'
+end
 class WebRobots
   # Creates a WebRobots object for a robot named +user_agent+, with

data/lib/webrobots/nokogiri.rb ADDED Viewed

@@ -0,0 +1,32 @@
+require 'nokogiri'
+class Nokogiri::HTML::Document
+  # Returns an array of lower-cased <meta name="ROBOTS"> tokens.  If
+  # no tag is found, returns an empty array.  An optional
+  # +custom_name+ specifies the name of a meta tag to look for ahead
+  # of "ROBOTS".  Names are compared in a case-insensitive manner.
+  def meta_robots(custom_name = nil)
+    (@meta_robots ||= {})[custom_name] =
+      (custom_name && parse_meta_robots(custom_name)) || parse_meta_robots('robots')
+  end
+  # Equivalent to meta_robots(custom_name).include?('noindex').
+  def noindex?(custom_name = nil)
+    meta_robots(custom_name).include?('noindex')
+  end
+  # Equivalent to meta_robots(custom_name).include?('nofollow').
+  def nofollow?(custom_name = nil)
+    meta_robots(custom_name).include?('nofollow')
+  end
+  private
+  def parse_meta_robots(custom_name)
+    pattern = /\A#{Regexp.quote(custom_name)}\z/i
+    meta = css('meta[@name]').find { |meta|
+      meta['name'].match(pattern)
+    } and content = meta['content'] or return []
+    content.downcase.split(/[,\s]+/)
+  end
+end

data/test/test_webrobots.rb CHANGED Viewed

@@ -283,9 +283,38 @@ Disallow: /
         assert !@testbot.allowed?("http://store.apple.com/vieworder")
         assert @msnbot.allowed?("http://store.apple.com/vieworder")
       }
-#      assert_nothing_raised {
+      assert_nothing_raised {
         assert !@testbot.allowed?("http://github.com/login")
-#      }
+      }
+    end
+  end
+  context "meta robots tag" do
+    setup do
+      @doc = Nokogiri::HTML(<<-HTML)
+<html>
+  <head>
+    <meta name="ROBOTS" content="NOFOLLOW">
+    <meta name="Slurp" content="noindex,nofollow">
+    <meta name="googlebot" content="noarchive, noindex">
+  </head>
+  <body>
+    test
+  </body>
+</html>
+      HTML
+    end
+    should "be properly parsed when given in HTML string" do
+      assert !@doc.noindex?
+      assert  @doc.nofollow?
+      assert  @doc.noindex?('slurp')
+      assert  @doc.nofollow?('slurp')
+      assert  @doc.noindex?('googlebot')
+      assert !@doc.nofollow?('googlebot')
+      assert  @doc.meta_robots('googlebot').include?('noarchive')
     end
   end
 end

data/webrobots.gemspec CHANGED Viewed

@@ -5,11 +5,11 @@
 Gem::Specification.new do |s|
   s.name = %q{webrobots}
-  s.version = "0.0.2"
+  s.version = "0.0.3"
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["Akinori MUSHA"]
-  s.date = %q{2011-01-03}
+  s.date = %q{2011-01-05}
   s.description = %q{This library helps write robots.txt compliant web robots in Ruby.
 }
   s.email = %q{knu@idaemons.org}
@@ -26,6 +26,7 @@ Gem::Specification.new do |s|
     "Rakefile",
     "VERSION",
     "lib/webrobots.rb",
+    "lib/webrobots/nokogiri.rb",
     "lib/webrobots/robotstxt.rb",
     "lib/webrobots/robotstxt.ry",
     "test/helper.rb",
@@ -46,23 +47,29 @@ Gem::Specification.new do |s|
     if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then
       s.add_runtime_dependency(%q<racc>, [">= 0"])
+      s.add_runtime_dependency(%q<nokogiri>, [">= 1.4.4"])
       s.add_development_dependency(%q<shoulda>, [">= 0"])
       s.add_development_dependency(%q<bundler>, ["~> 1.0.0"])
       s.add_development_dependency(%q<jeweler>, ["~> 1.5.1"])
       s.add_development_dependency(%q<rcov>, [">= 0"])
+      s.add_development_dependency(%q<racc>, [">= 0"])
     else
       s.add_dependency(%q<racc>, [">= 0"])
+      s.add_dependency(%q<nokogiri>, [">= 1.4.4"])
       s.add_dependency(%q<shoulda>, [">= 0"])
       s.add_dependency(%q<bundler>, ["~> 1.0.0"])
       s.add_dependency(%q<jeweler>, ["~> 1.5.1"])
       s.add_dependency(%q<rcov>, [">= 0"])
+      s.add_dependency(%q<racc>, [">= 0"])
     end
   else
     s.add_dependency(%q<racc>, [">= 0"])
+    s.add_dependency(%q<nokogiri>, [">= 1.4.4"])
     s.add_dependency(%q<shoulda>, [">= 0"])
     s.add_dependency(%q<bundler>, ["~> 1.0.0"])
     s.add_dependency(%q<jeweler>, ["~> 1.5.1"])
     s.add_dependency(%q<rcov>, [">= 0"])
+    s.add_dependency(%q<racc>, [">= 0"])
   end
 end

metadata CHANGED Viewed

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification
 name: webrobots
 version: !ruby/object:Gem::Version
-  hash: 27
+  hash: 25
   prerelease:
   segments:
   - 0
   - 0
-  - 2
-  version: 0.0.2
+  - 3
+  version: 0.0.3
 platform: ruby
 authors:
 - Akinori MUSHA
@@ -15,7 +15,7 @@ autorequire:
 bindir: bin
 cert_chain: []
-date: 2011-01-03 00:00:00 +09:00
+date: 2011-01-05 00:00:00 +09:00
 default_executable:
 dependencies:
 - !ruby/object:Gem::Dependency
@@ -33,8 +33,24 @@ dependencies:
   prerelease: false
   name: racc
 - !ruby/object:Gem::Dependency
-  type: :development
+  type: :runtime
   version_requirements: &id002 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        hash: 15
+        segments:
+        - 1
+        - 4
+        - 4
+        version: 1.4.4
+  requirement: *id002
+  prerelease: false
+  name: nokogiri
+- !ruby/object:Gem::Dependency
+  type: :development
+  version_requirements: &id003 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ">="
@@ -43,12 +59,12 @@ dependencies:
         segments:
         - 0
         version: "0"
-  requirement: *id002
+  requirement: *id003
   prerelease: false
   name: shoulda
 - !ruby/object:Gem::Dependency
   type: :development
-  version_requirements: &id003 !ruby/object:Gem::Requirement
+  version_requirements: &id004 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -59,12 +75,12 @@ dependencies:
         - 0
         - 0
         version: 1.0.0
-  requirement: *id003
+  requirement: *id004
   prerelease: false
   name: bundler
 - !ruby/object:Gem::Dependency
   type: :development
-  version_requirements: &id004 !ruby/object:Gem::Requirement
+  version_requirements: &id005 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -75,12 +91,12 @@ dependencies:
         - 5
         - 1
         version: 1.5.1
-  requirement: *id004
+  requirement: *id005
   prerelease: false
   name: jeweler
 - !ruby/object:Gem::Dependency
   type: :development
-  version_requirements: &id005 !ruby/object:Gem::Requirement
+  version_requirements: &id006 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ">="
@@ -89,9 +105,23 @@ dependencies:
         segments:
         - 0
         version: "0"
-  requirement: *id005
+  requirement: *id006
   prerelease: false
   name: rcov
+- !ruby/object:Gem::Dependency
+  type: :development
+  version_requirements: &id007 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        hash: 3
+        segments:
+        - 0
+        version: "0"
+  requirement: *id007
+  prerelease: false
+  name: racc
 description: |
   This library helps write robots.txt compliant web robots in Ruby.
@@ -112,6 +142,7 @@ files:
 - Rakefile
 - VERSION
 - lib/webrobots.rb
+- lib/webrobots/nokogiri.rb
 - lib/webrobots/robotstxt.rb
 - lib/webrobots/robotstxt.ry
 - test/helper.rb