robocop 0.1.0

data/.gitignore

@@ -0,0 +1,17 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp

data/.rbenv-version

@@ -0,0 +1 @@
+1.9.3-p125

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in robocop.gemspec
+gemspec

data/Guardfile

@@ -0,0 +1,5 @@
+guard 'rspec', :cli => '--colour --format Fuubar --drb', :all_on_start => false, :all_after_pass => false do
+  watch('spec/spec_helper.rb')                       { "spec" }
+  watch(%r{^spec/.+_spec\.rb})
+  watch(%r{^lib/(.+)\.rb})                           { |m| "spec/lib/#{m[1]}_spec.rb" }
+end

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2012 PJ Kelly
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,85 @@
+# Robocop
+
+Robocop is a simple Rack middleware that inserts the `X-Robots-Tag` into the headers of all your responses.
+
+The `X-Robots-Tag` can be used in place of a `robots.txt` file or meta tags to tell crawlers what they're allowed to do with your content. See [this article](https://developers.google.com/webmasters/control-crawl-index/docs/robots_meta_tag) for more information.
+
+## Installation
+
+The simplest way to install Robocop is to use [Bundler](http://gembundler.com/).
+
+Add Robocop to your `Gemfile`:
+
+``` ruby
+gem 'robocop'
+```
+
+## Basic Usage
+
+### Rails
+
+To use Robocop in your Rails application, add the following line to your application config file (`config/application.rb` for Rails 3, `config/environment.rb` for Rails 2):
+
+``` ruby
+config.middleware.use Robocop, :directives => %w(all)
+```
+
+### Other Rack Applications (Sinatra, Padrino, etc.)
+
+Simple add the following to your `config.ru`:
+
+``` ruby
+use Robocop, :directives => %w(all)
+```
+
+## Options
+
+The following directives can be passed in to Robocop's configuration:
+
+* all
+* noindex
+* nofollow
+* none
+* noarchive
+* nosnippet
+* noodp
+* notranslate
+* noimageindex
+
+### Directives (useragent agnostic)
+
+If you just want to specify a list of directives for all useragents to follow, simply pass in an array of directives with the `:directive` option:
+
+``` ruby
+config.middleware.use Robocop, :directives => %w(noindex nofollow)
+```
+
+### Useragents
+
+If you want to give specific user agents unique sets of directives, you can do so by passing in the `:useragent` option:
+
+``` ruby
+config.middleware.use Robocop, :useragents => {
+  :googlebot => %w(noindex nofollow, noimageindex),
+  :otherbot => %w(none)
+}
+```
+
+It should be noted that if both the `:useragents` and `:directives` options are passed in, the `:useragents` option takes precedence.
+
+## TODO
+
+* Add support for `unavailable_after` directive.
+* Sanity checks for directives that are passed in. e.g. passing all, noindex, nofollow doesn't make any sense.
+
+## Note on Patches / Pull Requests
+
+* Fork the project.
+* Code your feature addition or bug fix.
+* **Add specs for it.** This is important so we don't break it in a future version unintentionally.
+* Commit, do not mess with Rakefile or version number. If you want to have your own version, that's fine but bump version in a commit by itself so we can ignore it when merging.
+* Send a pull request. Bonus points for topic branches.
+
+## Copyright
+
+Copyright (c) 2012 PJ Kelly. See LICENSE for details.

data/Rakefile

@@ -0,0 +1,8 @@
+#!/usr/bin/env rake
+
+require "bundler/gem_tasks"
+require 'rspec/core/rake_task'
+
+RSpec::Core::RakeTask.new(:spec)
+
+task :default => :spec

data/lib/robocop.rb

@@ -0,0 +1 @@
+require "robocop/middleware"

data/lib/robocop/middleware.rb

@@ -0,0 +1,52 @@
+module Robocop
+  class Middleware
+    VALID_DIRECTIVES = %w(
+      all
+      noindex
+      nofollow
+      none
+      noarchive
+      nosnippet
+      noodp
+      notranslate
+      noimageindex
+    )
+
+    def initialize(app, options = {})
+      @app, = app
+      if options[:useragents]
+        @useragents = options[:useragents]
+      elsif options[:directives]
+        @directives = options[:directives]
+      else
+        @ignore = true
+      end
+    end
+
+    def call(env)
+      if ignore?
+        @app.call(env)
+      else
+        status, headers, body = @app.call(env)
+        add_robots_tag_header!(headers)
+        [status, headers, body]
+      end
+    end
+
+    def ignore?
+      @ignore
+    end
+
+    def add_robots_tag_header!(headers)
+      if @useragents
+        headers['X-Robots-Tag'] = @useragents.collect { |useragent, directives| [useragent, valid_directives(directives).join(', ')].join(': ') }.join("\n")
+      else
+        headers['X-Robots-Tag'] = valid_directives(@directives).join(', ')
+      end
+    end
+
+    def valid_directives(directives)
+      directives.reject { |d| !VALID_DIRECTIVES.include?(d) }
+    end
+  end
+end

data/robocop.gemspec

@@ -0,0 +1,24 @@
+# -*- encoding: utf-8 -*-
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+
+Gem::Specification.new do |gem|
+  gem.name          = "robocop"
+  gem.version       = "0.1.0"
+  gem.authors       = ["PJ Kelly"]
+  gem.email         = ["me@pjkel.ly"]
+  gem.description   = %q{Rack middleware that inserts the X-Robots-Tag into all responses.}
+  gem.summary       = %q{Rack middleware that inserts the X-Robots-Tag into all responses.}
+  gem.homepage      = "https://github.com/pjkelly/robocop"
+
+  gem.files         = `git ls-files`.split($/)
+  gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
+  gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
+  gem.require_paths = ["lib"]
+  gem.add_dependency 'rack'
+  gem.add_development_dependency 'rspec'
+  gem.add_development_dependency 'guard-rspec'
+  gem.add_development_dependency 'fuubar'
+  gem.add_development_dependency 'rb-fsevent'
+  gem.add_development_dependency 'terminal-notifier-guard'
+end

data/spec/lib/robocop/middleware_spec.rb

@@ -0,0 +1,60 @@
+require 'spec_helper'
+
+describe Robocop::Middleware do
+  let(:fake_request) { lambda { |env| [200, {}, []] } }
+  let(:middleware) { Robocop::Middleware.new(fake_request) }
+  let(:mock_request) { Rack::MockRequest.new(middleware) }
+  let(:response) { mock_request.get("/") }
+  let(:valid_directives) { %w(noindex nofollow) }
+  let(:invalid_directives) { %w(dogs cats) }
+
+  context "when no options are passed in" do
+    it("should pass the request along") { response.status.should == 200 }
+  end
+
+  context "when useragents are passed in" do
+    let(:useragents) {
+      {
+        :googlebot => %w(noindex nofollow),
+        :otherbot => %w(noindex nofollow)
+      }
+    }
+    let(:valid_header_string) { "googlebot: noindex, nofollow\notherbot: noindex, nofollow" }
+    let(:middleware) { Robocop::Middleware.new(fake_request, :useragents => useragents) }
+
+    it "should return those useragents and their directives in the X-Robots-Tag header" do
+      response.headers["X-Robots-Tag"].should == valid_header_string
+    end
+
+    context "and some of their directives are invalid" do
+      let(:useragents) {
+        {
+          :googlebot => (valid_directives + invalid_directives),
+          :otherbot => (valid_directives + invalid_directives)
+        }
+      }
+
+      it "should return only valid directives for each of the useragents in the X-Robots-Tag header" do
+        response.headers["X-Robots-Tag"].should == valid_header_string
+      end
+    end
+  end
+
+  context "when directives are passed in" do
+    let(:directives) { %w(noindex nofollow) }
+    let(:middleware) { Robocop::Middleware.new(fake_request, :directives => directives) }
+
+    it "should return those directives in the X-Robots-Tag header" do
+      response.headers["X-Robots-Tag"].should == directives.join(', ')
+    end
+
+    context "and some of the directives are invalid" do
+      let(:directives) { (valid_directives + invalid_directives) }
+
+      it "should return only valid directives in the X-Robots-Tag header" do
+        response.headers["X-Robots-Tag"].should == valid_directives.join(', ')
+      end
+    end
+  end
+end
+

data/spec/spec_helper.rb

@@ -0,0 +1,9 @@
+require 'bundler/setup'
+require 'rspec'
+require 'rack'
+require 'robocop'
+
+Dir[File.expand_path('../support/**/*', __FILE__)].each { |f| require f }
+
+RSpec.configure do |config|
+end

metadata

@@ -0,0 +1,126 @@
+--- !ruby/object:Gem::Specification
+name: robocop
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+  prerelease: 
+platform: ruby
+authors:
+- PJ Kelly
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2012-11-23 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: rack
+  requirement: &70362405142460 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: *70362405142460
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: &70362405141740 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: *70362405141740
+- !ruby/object:Gem::Dependency
+  name: guard-rspec
+  requirement: &70362405141180 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: *70362405141180
+- !ruby/object:Gem::Dependency
+  name: fuubar
+  requirement: &70362405140120 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: *70362405140120
+- !ruby/object:Gem::Dependency
+  name: rb-fsevent
+  requirement: &70362405156020 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: *70362405156020
+- !ruby/object:Gem::Dependency
+  name: terminal-notifier-guard
+  requirement: &70362405155060 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: *70362405155060
+description: Rack middleware that inserts the X-Robots-Tag into all responses.
+email:
+- me@pjkel.ly
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- .rbenv-version
+- Gemfile
+- Guardfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- lib/robocop.rb
+- lib/robocop/middleware.rb
+- robocop.gemspec
+- spec/lib/robocop/middleware_spec.rb
+- spec/spec_helper.rb
+homepage: https://github.com/pjkelly/robocop
+licenses: []
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.11
+signing_key: 
+specification_version: 3
+summary: Rack middleware that inserts the X-Robots-Tag into all responses.
+test_files:
+- spec/lib/robocop/middleware_spec.rb
+- spec/spec_helper.rb
+has_rdoc: