better_robots 0.0.1

data/.gitignore

@@ -0,0 +1 @@
+Gemfile.lock

data/Gemfile

@@ -0,0 +1,19 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in better_robots.gemspec
+gemspec
+
+group :development, :test do
+  gem 'rake'
+  gem 'sinatra'
+  gem 'minitest'
+  gem 'test-unit'
+  gem 'test-spec'
+  gem 'rack-test'
+  gem 'simplecov'
+  gem 'guard'
+  gem 'guard-markdown'
+  gem 'guard-minitest'
+  gem "rb-fsevent"
+  gem "growl"
+end

data/MIT-LICENSE

@@ -0,0 +1,22 @@
+Copyright (c) 2012 Gerry Power <gerry@thepowerhouse.com>
+
+Permission is hereby granted, free of charge, to any person
+obtaining a copy of this software and associated documentation
+files (the "Software"), to deal in the Software without
+restriction, including without limitation the rights to use,
+copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the
+Software is furnished to do so, subject to the following
+conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,80 @@
+# BetterRobots - Better SEO with robots.txt
+
+BetterRobots is designed to SEO enhance your robots.txt serving, in particular for situations where
+you have multiple domains or subdomains being served from one application. e.g. www.yoursite.com,
+assets.yoursite.com, beta.yoursite.com, etc.
+
+## SEO and Your robots.txt
+
+A web application commonly has several subdomains that refer to the same application. For example,
+a standard practice to speed browser page load time, is to have multiple asset hosts aliased to your
+application, allowing a browser to simultaneously stream multiple assets. In an SEO context, each of 
+these aliased hosts are considered to be duplicate content. To avoid this, you should have different
+robots.txt that exclude search engines for all but your canonical domain.
+
+BetterRobots provides a robots.txt configuration for your conical domain, optional robots.txt for others,
+and defaults to disallow for everything that does not have a matching robots.txt.
+
+As a quick example, assume your canonical domain is www.yoursite.com with a subdomain 'assets0' and
+'api'. If you rename your robots.txt to www.yoursite.com.robots.txt, the following get requests for
+robots.txt will return the following:
+
+    www.yoursite.com/robots.txt -> User-agent: *
+                                   Crawl-Delay: 3
+                               
+    assets0.yoursite.com/robots.txt -> User-agent: *
+                                           Disallow: /
+
+    api.yoursite.com/robots.txt -> User-agent: *
+                                   Disallow: /
+
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+    gem 'better_robots'
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install better_robots
+
+## Usage
+
+For Rails 3, add a route to config/routes.rb
+```ruby
+match "/robots.txt" => BetterRobots::Generator
+```
+
+For each domain name that you want a robots.txt file served, rename your public/robots.txt to
+the *\<fully qualified domain name\>*.robots.txt. e.g. Domain: www.example.org; public/www.example.org.robots.txt
+All other domain names will default to:
+    User-agent: *
+    Disallow: /
+
+## <a name="works_with"></a>Works with:
+
+BetterRobots is a Rack based app, and should work with any Rack compatible framework. It has been tested with
+Rails 3.2 and Sinatra 1.3, and on the following Ruby implementations:
+
+* JRuby 1.7.1
+* MRI 1.8.7
+* MRI 1.9.2
+* MRI 1.9.3
+* Rubinius 1.2.4
+* Ruby EE 1.8.7
+
+### License
+
+webhookr is released under the [MIT license](http://www.opensource.org/licenses/MIT).
+
+## Author
+
+* [Gerry Power](https://github.com/gerrypower)
+
+
+

data/Rakefile

@@ -0,0 +1,13 @@
+require "bundler/gem_tasks"
+require 'rake/testtask'
+
+$:.push File.expand_path(File.dirname(__FILE__), 'lib')
+
+desc 'Test better_robots'
+Rake::TestTask.new(:test) do |t|
+  t.test_files = FileList['test/**/*_*spec.rb']
+  t.verbose = !!ENV['VERBOSE_TESTS']
+  t.warning = !!ENV['WARNINGS']
+end
+
+task :default => :test

data/better_robots.gemspec

@@ -0,0 +1,19 @@
+# -*- encoding: utf-8 -*-
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'better_robots/version'
+
+Gem::Specification.new do |gem|
+  gem.name          = "better_robots"
+  gem.version       = BetterRobots::VERSION
+  gem.authors       = ["Gerry Power"]
+  gem.email         = ["gerry@thepowerhouse.com"]
+  gem.description   = "Better SEO with robots.txt"
+  gem.summary       = gem.description
+  gem.homepage      = "https://github.com/gerrypower/better_robots"
+
+  gem.files         = `git ls-files`.split($/)
+  gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
+  gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
+  gem.require_paths = ["lib"]
+end

data/lib/better_robots/version.rb

@@ -0,0 +1,3 @@
+module BetterRobots
+  VERSION = "0.0.1"
+end

data/lib/better_robots.rb

@@ -0,0 +1,52 @@
+require "better_robots/version"
+require 'ostruct'
+
+module BetterRobots
+  class Generator
+    DEFAULT_DISALLOW_TEXT = "User-agent: *\nDisallow: /"
+    ROBOTS_CACHE = {}
+
+    class << self
+
+      def config
+        @config ||= {
+          :robots_txt_path => ((Rails.root.join("public") if defined? Rails) || ".")
+        }
+      end
+
+      def call(env)
+        res = cached_robots_txt_for(env['SERVER_NAME'])
+        [ 200, headers_for(res), [ res.txt ] ]
+      end
+
+      def cached_robots_txt_for(server_name)
+        ROBOTS_CACHE[server_name] ||= robots_txt_for(server_name)
+      end
+
+      def robots_txt_for(server_name)
+        begin
+          txt = read_robots_file(server_name)
+          OpenStruct.new(:txt => txt, :length => txt.length)
+        rescue Errno::ENOENT
+          OpenStruct.new(:txt => DEFAULT_DISALLOW_TEXT, :length => DEFAULT_DISALLOW_TEXT.length)
+        end
+      end
+
+      def read_robots_file(server_name)
+        File.read(File.join(config[:robots_txt_path], "#{server_name}.robots.txt"))
+      end
+
+      def headers_for(res)
+        {
+          "Content-Type" => "text/plain",
+          "Content-Length" => res.length.to_s
+        }
+      end
+
+    end
+
+    private_class_method :cached_robots_txt_for, :robots_txt_for,
+                         :read_robots_file, :headers_for
+
+  end
+end

data/test/integration/better_robots_spec.rb

@@ -0,0 +1,61 @@
+$: << File.join(File.dirname(__FILE__), "..")
+
+require 'routes'
+require 'test_helper'
+require 'rack/test'
+
+describe BetterRobots::Generator do
+
+  def get_robots(host = "site.org")
+    BetterRobots::Generator.config[:robots_txt_path] = File.join(File.dirname(__FILE__), "robots")
+    browser = Rack::Test::Session.new(Rack::MockSession.new(Sinatra::Application, host))    
+    browser.get '/robots.txt'
+    browser
+  end
+
+  def robot_file_contents(robot_file)
+    File.read(File.join(BetterRobots::Generator.config[:robots_txt_path], robot_file))
+  end
+
+  def reload_better_robots
+    Object.send(:remove_const, :BetterRobots).send(:remove_const, :Generator)
+    load 'better_robots.rb'
+  end
+
+  it "should process /robots.txt" do
+    get_robots.last_response.should.be.ok
+  end
+
+  it "should display the contents of the test.site.org.robots.txt" do
+    get_robots("test.site.org").last_response.body.should.equal \
+      robot_file_contents("test.site.org.robots.txt")
+  end
+
+  it "should have a content type of text" do
+    get_robots.last_response.headers["Content-Type"].should.equal "text/plain"
+  end
+
+  it "should have a content length matching the robots file length" do
+    get_robots("test.site.org").last_response.headers["Content-Length"].should.equal \
+      robot_file_contents("test.site.org.robots.txt").length.to_s
+  end
+
+  it "should default to disallow if there is no robots file" do
+    get_robots("test.nosite.org").last_response.body.should.equal \
+      BetterRobots::Generator::DEFAULT_DISALLOW_TEXT
+  end
+
+  it "should cache the results" do
+    get_robots("test.site.org")
+    BetterRobots::Generator::ROBOTS_CACHE["test.site.org"].txt.should.equal \
+      robot_file_contents("test.site.org.robots.txt")
+  end
+
+  it "should use Rails.root + public for robot files, if Rails is defined" do
+    Rails = OpenStruct.new({:root => Pathname.new(File.dirname(__FILE__))})
+    reload_better_robots
+    BetterRobots::Generator.config[:robots_txt_path].to_s.should.equal \
+      File.join(File.dirname(__FILE__), "public")
+  end
+
+end

data/test/integration/robots/test.site.org.robots.txt

@@ -0,0 +1,5 @@
+User-agent: Google
+Disallow:
+
+User-agent: *
+Disallow: /

data/test/routes.rb

@@ -0,0 +1,6 @@
+require 'sinatra'
+require 'better_robots'
+
+get '/robots.txt' do
+  BetterRobots::Generator.call(env)
+end

data/test/test_helper.rb

@@ -0,0 +1,10 @@
+require 'simplecov'
+SimpleCov.start
+
+require 'minitest/spec'
+require 'minitest/autorun'
+require 'better_robots'
+
+require 'test/spec'
+
+set :environment, :test

metadata

@@ -0,0 +1,61 @@
+--- !ruby/object:Gem::Specification
+name: better_robots
+version: !ruby/object:Gem::Version
+  prerelease: 
+  version: 0.0.1
+platform: ruby
+authors:
+- Gerry Power
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2012-12-24 00:00:00.000000000 Z
+dependencies: []
+description: Better SEO with robots.txt
+email:
+- gerry@thepowerhouse.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- Gemfile
+- MIT-LICENSE
+- README.md
+- Rakefile
+- better_robots.gemspec
+- lib/better_robots.rb
+- lib/better_robots/version.rb
+- test/integration/better_robots_spec.rb
+- test/integration/robots/test.site.org.robots.txt
+- test/routes.rb
+- test/test_helper.rb
+homepage: https://github.com/gerrypower/better_robots
+licenses: []
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+  none: false
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+  none: false
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.23
+signing_key: 
+specification_version: 3
+summary: Better SEO with robots.txt
+test_files:
+- test/integration/better_robots_spec.rb
+- test/integration/robots/test.site.org.robots.txt
+- test/routes.rb
+- test/test_helper.rb