robocop 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,17 @@
1
+ *.gem
2
+ *.rbc
3
+ .bundle
4
+ .config
5
+ .yardoc
6
+ Gemfile.lock
7
+ InstalledFiles
8
+ _yardoc
9
+ coverage
10
+ doc/
11
+ lib/bundler/man
12
+ pkg
13
+ rdoc
14
+ spec/reports
15
+ test/tmp
16
+ test/version_tmp
17
+ tmp
@@ -0,0 +1 @@
1
+ 1.9.3-p125
data/Gemfile ADDED
@@ -0,0 +1,4 @@
1
+ source 'https://rubygems.org'
2
+
3
+ # Specify your gem's dependencies in robocop.gemspec
4
+ gemspec
@@ -0,0 +1,5 @@
1
+ guard 'rspec', :cli => '--colour --format Fuubar --drb', :all_on_start => false, :all_after_pass => false do
2
+ watch('spec/spec_helper.rb') { "spec" }
3
+ watch(%r{^spec/.+_spec\.rb})
4
+ watch(%r{^lib/(.+)\.rb}) { |m| "spec/lib/#{m[1]}_spec.rb" }
5
+ end
@@ -0,0 +1,22 @@
1
+ Copyright (c) 2012 PJ Kelly
2
+
3
+ MIT License
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining
6
+ a copy of this software and associated documentation files (the
7
+ "Software"), to deal in the Software without restriction, including
8
+ without limitation the rights to use, copy, modify, merge, publish,
9
+ distribute, sublicense, and/or sell copies of the Software, and to
10
+ permit persons to whom the Software is furnished to do so, subject to
11
+ the following conditions:
12
+
13
+ The above copyright notice and this permission notice shall be
14
+ included in all copies or substantial portions of the Software.
15
+
16
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
17
+ EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
18
+ MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
19
+ NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
20
+ LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
21
+ OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
22
+ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
@@ -0,0 +1,85 @@
1
+ # Robocop
2
+
3
+ Robocop is a simple Rack middleware that inserts the `X-Robots-Tag` into the headers of all your responses.
4
+
5
+ The `X-Robots-Tag` can be used in place of a `robots.txt` file or meta tags to tell crawlers what they're allowed to do with your content. See [this article](https://developers.google.com/webmasters/control-crawl-index/docs/robots_meta_tag) for more information.
6
+
7
+ ## Installation
8
+
9
+ The simplest way to install Robocop is to use [Bundler](http://gembundler.com/).
10
+
11
+ Add Robocop to your `Gemfile`:
12
+
13
+ ``` ruby
14
+ gem 'robocop'
15
+ ```
16
+
17
+ ## Basic Usage
18
+
19
+ ### Rails
20
+
21
+ To use Robocop in your Rails application, add the following line to your application config file (`config/application.rb` for Rails 3, `config/environment.rb` for Rails 2):
22
+
23
+ ``` ruby
24
+ config.middleware.use Robocop, :directives => %w(all)
25
+ ```
26
+
27
+ ### Other Rack Applications (Sinatra, Padrino, etc.)
28
+
29
+ Simple add the following to your `config.ru`:
30
+
31
+ ``` ruby
32
+ use Robocop, :directives => %w(all)
33
+ ```
34
+
35
+ ## Options
36
+
37
+ The following directives can be passed in to Robocop's configuration:
38
+
39
+ * all
40
+ * noindex
41
+ * nofollow
42
+ * none
43
+ * noarchive
44
+ * nosnippet
45
+ * noodp
46
+ * notranslate
47
+ * noimageindex
48
+
49
+ ### Directives (useragent agnostic)
50
+
51
+ If you just want to specify a list of directives for all useragents to follow, simply pass in an array of directives with the `:directive` option:
52
+
53
+ ``` ruby
54
+ config.middleware.use Robocop, :directives => %w(noindex nofollow)
55
+ ```
56
+
57
+ ### Useragents
58
+
59
+ If you want to give specific user agents unique sets of directives, you can do so by passing in the `:useragent` option:
60
+
61
+ ``` ruby
62
+ config.middleware.use Robocop, :useragents => {
63
+ :googlebot => %w(noindex nofollow, noimageindex),
64
+ :otherbot => %w(none)
65
+ }
66
+ ```
67
+
68
+ It should be noted that if both the `:useragents` and `:directives` options are passed in, the `:useragents` option takes precedence.
69
+
70
+ ## TODO
71
+
72
+ * Add support for `unavailable_after` directive.
73
+ * Sanity checks for directives that are passed in. e.g. passing all, noindex, nofollow doesn't make any sense.
74
+
75
+ ## Note on Patches / Pull Requests
76
+
77
+ * Fork the project.
78
+ * Code your feature addition or bug fix.
79
+ * **Add specs for it.** This is important so we don't break it in a future version unintentionally.
80
+ * Commit, do not mess with Rakefile or version number. If you want to have your own version, that's fine but bump version in a commit by itself so we can ignore it when merging.
81
+ * Send a pull request. Bonus points for topic branches.
82
+
83
+ ## Copyright
84
+
85
+ Copyright (c) 2012 PJ Kelly. See LICENSE for details.
@@ -0,0 +1,8 @@
1
+ #!/usr/bin/env rake
2
+
3
+ require "bundler/gem_tasks"
4
+ require 'rspec/core/rake_task'
5
+
6
+ RSpec::Core::RakeTask.new(:spec)
7
+
8
+ task :default => :spec
@@ -0,0 +1 @@
1
+ require "robocop/middleware"
@@ -0,0 +1,52 @@
1
+ module Robocop
2
+ class Middleware
3
+ VALID_DIRECTIVES = %w(
4
+ all
5
+ noindex
6
+ nofollow
7
+ none
8
+ noarchive
9
+ nosnippet
10
+ noodp
11
+ notranslate
12
+ noimageindex
13
+ )
14
+
15
+ def initialize(app, options = {})
16
+ @app, = app
17
+ if options[:useragents]
18
+ @useragents = options[:useragents]
19
+ elsif options[:directives]
20
+ @directives = options[:directives]
21
+ else
22
+ @ignore = true
23
+ end
24
+ end
25
+
26
+ def call(env)
27
+ if ignore?
28
+ @app.call(env)
29
+ else
30
+ status, headers, body = @app.call(env)
31
+ add_robots_tag_header!(headers)
32
+ [status, headers, body]
33
+ end
34
+ end
35
+
36
+ def ignore?
37
+ @ignore
38
+ end
39
+
40
+ def add_robots_tag_header!(headers)
41
+ if @useragents
42
+ headers['X-Robots-Tag'] = @useragents.collect { |useragent, directives| [useragent, valid_directives(directives).join(', ')].join(': ') }.join("\n")
43
+ else
44
+ headers['X-Robots-Tag'] = valid_directives(@directives).join(', ')
45
+ end
46
+ end
47
+
48
+ def valid_directives(directives)
49
+ directives.reject { |d| !VALID_DIRECTIVES.include?(d) }
50
+ end
51
+ end
52
+ end
@@ -0,0 +1,24 @@
1
+ # -*- encoding: utf-8 -*-
2
+ lib = File.expand_path('../lib', __FILE__)
3
+ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
4
+
5
+ Gem::Specification.new do |gem|
6
+ gem.name = "robocop"
7
+ gem.version = "0.1.0"
8
+ gem.authors = ["PJ Kelly"]
9
+ gem.email = ["me@pjkel.ly"]
10
+ gem.description = %q{Rack middleware that inserts the X-Robots-Tag into all responses.}
11
+ gem.summary = %q{Rack middleware that inserts the X-Robots-Tag into all responses.}
12
+ gem.homepage = "https://github.com/pjkelly/robocop"
13
+
14
+ gem.files = `git ls-files`.split($/)
15
+ gem.executables = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
16
+ gem.test_files = gem.files.grep(%r{^(test|spec|features)/})
17
+ gem.require_paths = ["lib"]
18
+ gem.add_dependency 'rack'
19
+ gem.add_development_dependency 'rspec'
20
+ gem.add_development_dependency 'guard-rspec'
21
+ gem.add_development_dependency 'fuubar'
22
+ gem.add_development_dependency 'rb-fsevent'
23
+ gem.add_development_dependency 'terminal-notifier-guard'
24
+ end
@@ -0,0 +1,60 @@
1
+ require 'spec_helper'
2
+
3
+ describe Robocop::Middleware do
4
+ let(:fake_request) { lambda { |env| [200, {}, []] } }
5
+ let(:middleware) { Robocop::Middleware.new(fake_request) }
6
+ let(:mock_request) { Rack::MockRequest.new(middleware) }
7
+ let(:response) { mock_request.get("/") }
8
+ let(:valid_directives) { %w(noindex nofollow) }
9
+ let(:invalid_directives) { %w(dogs cats) }
10
+
11
+ context "when no options are passed in" do
12
+ it("should pass the request along") { response.status.should == 200 }
13
+ end
14
+
15
+ context "when useragents are passed in" do
16
+ let(:useragents) {
17
+ {
18
+ :googlebot => %w(noindex nofollow),
19
+ :otherbot => %w(noindex nofollow)
20
+ }
21
+ }
22
+ let(:valid_header_string) { "googlebot: noindex, nofollow\notherbot: noindex, nofollow" }
23
+ let(:middleware) { Robocop::Middleware.new(fake_request, :useragents => useragents) }
24
+
25
+ it "should return those useragents and their directives in the X-Robots-Tag header" do
26
+ response.headers["X-Robots-Tag"].should == valid_header_string
27
+ end
28
+
29
+ context "and some of their directives are invalid" do
30
+ let(:useragents) {
31
+ {
32
+ :googlebot => (valid_directives + invalid_directives),
33
+ :otherbot => (valid_directives + invalid_directives)
34
+ }
35
+ }
36
+
37
+ it "should return only valid directives for each of the useragents in the X-Robots-Tag header" do
38
+ response.headers["X-Robots-Tag"].should == valid_header_string
39
+ end
40
+ end
41
+ end
42
+
43
+ context "when directives are passed in" do
44
+ let(:directives) { %w(noindex nofollow) }
45
+ let(:middleware) { Robocop::Middleware.new(fake_request, :directives => directives) }
46
+
47
+ it "should return those directives in the X-Robots-Tag header" do
48
+ response.headers["X-Robots-Tag"].should == directives.join(', ')
49
+ end
50
+
51
+ context "and some of the directives are invalid" do
52
+ let(:directives) { (valid_directives + invalid_directives) }
53
+
54
+ it "should return only valid directives in the X-Robots-Tag header" do
55
+ response.headers["X-Robots-Tag"].should == valid_directives.join(', ')
56
+ end
57
+ end
58
+ end
59
+ end
60
+
@@ -0,0 +1,9 @@
1
+ require 'bundler/setup'
2
+ require 'rspec'
3
+ require 'rack'
4
+ require 'robocop'
5
+
6
+ Dir[File.expand_path('../support/**/*', __FILE__)].each { |f| require f }
7
+
8
+ RSpec.configure do |config|
9
+ end
metadata ADDED
@@ -0,0 +1,126 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: robocop
3
+ version: !ruby/object:Gem::Version
4
+ version: 0.1.0
5
+ prerelease:
6
+ platform: ruby
7
+ authors:
8
+ - PJ Kelly
9
+ autorequire:
10
+ bindir: bin
11
+ cert_chain: []
12
+ date: 2012-11-23 00:00:00.000000000 Z
13
+ dependencies:
14
+ - !ruby/object:Gem::Dependency
15
+ name: rack
16
+ requirement: &70362405142460 !ruby/object:Gem::Requirement
17
+ none: false
18
+ requirements:
19
+ - - ! '>='
20
+ - !ruby/object:Gem::Version
21
+ version: '0'
22
+ type: :runtime
23
+ prerelease: false
24
+ version_requirements: *70362405142460
25
+ - !ruby/object:Gem::Dependency
26
+ name: rspec
27
+ requirement: &70362405141740 !ruby/object:Gem::Requirement
28
+ none: false
29
+ requirements:
30
+ - - ! '>='
31
+ - !ruby/object:Gem::Version
32
+ version: '0'
33
+ type: :development
34
+ prerelease: false
35
+ version_requirements: *70362405141740
36
+ - !ruby/object:Gem::Dependency
37
+ name: guard-rspec
38
+ requirement: &70362405141180 !ruby/object:Gem::Requirement
39
+ none: false
40
+ requirements:
41
+ - - ! '>='
42
+ - !ruby/object:Gem::Version
43
+ version: '0'
44
+ type: :development
45
+ prerelease: false
46
+ version_requirements: *70362405141180
47
+ - !ruby/object:Gem::Dependency
48
+ name: fuubar
49
+ requirement: &70362405140120 !ruby/object:Gem::Requirement
50
+ none: false
51
+ requirements:
52
+ - - ! '>='
53
+ - !ruby/object:Gem::Version
54
+ version: '0'
55
+ type: :development
56
+ prerelease: false
57
+ version_requirements: *70362405140120
58
+ - !ruby/object:Gem::Dependency
59
+ name: rb-fsevent
60
+ requirement: &70362405156020 !ruby/object:Gem::Requirement
61
+ none: false
62
+ requirements:
63
+ - - ! '>='
64
+ - !ruby/object:Gem::Version
65
+ version: '0'
66
+ type: :development
67
+ prerelease: false
68
+ version_requirements: *70362405156020
69
+ - !ruby/object:Gem::Dependency
70
+ name: terminal-notifier-guard
71
+ requirement: &70362405155060 !ruby/object:Gem::Requirement
72
+ none: false
73
+ requirements:
74
+ - - ! '>='
75
+ - !ruby/object:Gem::Version
76
+ version: '0'
77
+ type: :development
78
+ prerelease: false
79
+ version_requirements: *70362405155060
80
+ description: Rack middleware that inserts the X-Robots-Tag into all responses.
81
+ email:
82
+ - me@pjkel.ly
83
+ executables: []
84
+ extensions: []
85
+ extra_rdoc_files: []
86
+ files:
87
+ - .gitignore
88
+ - .rbenv-version
89
+ - Gemfile
90
+ - Guardfile
91
+ - LICENSE.txt
92
+ - README.md
93
+ - Rakefile
94
+ - lib/robocop.rb
95
+ - lib/robocop/middleware.rb
96
+ - robocop.gemspec
97
+ - spec/lib/robocop/middleware_spec.rb
98
+ - spec/spec_helper.rb
99
+ homepage: https://github.com/pjkelly/robocop
100
+ licenses: []
101
+ post_install_message:
102
+ rdoc_options: []
103
+ require_paths:
104
+ - lib
105
+ required_ruby_version: !ruby/object:Gem::Requirement
106
+ none: false
107
+ requirements:
108
+ - - ! '>='
109
+ - !ruby/object:Gem::Version
110
+ version: '0'
111
+ required_rubygems_version: !ruby/object:Gem::Requirement
112
+ none: false
113
+ requirements:
114
+ - - ! '>='
115
+ - !ruby/object:Gem::Version
116
+ version: '0'
117
+ requirements: []
118
+ rubyforge_project:
119
+ rubygems_version: 1.8.11
120
+ signing_key:
121
+ specification_version: 3
122
+ summary: Rack middleware that inserts the X-Robots-Tag into all responses.
123
+ test_files:
124
+ - spec/lib/robocop/middleware_spec.rb
125
+ - spec/spec_helper.rb
126
+ has_rdoc: