human_power 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: b13d5ae833d0a82cf616bcf7cd7a41bd237acf38
+  data.tar.gz: 5f43b7c423e35346789c45968e778ac978aec87b
+SHA512:
+  metadata.gz: 5fd802c2b31a394473f34af52be0be9399215c30acfc3dc3b88779d41484b1fab2f526115f860d48832731a86d45b0411a820099471708e3b12cd1c13ecd23a8
+  data.tar.gz: ecf2616ef34b18622f3b4c9618e24eec259ea037520abb6af138808703e5af57347734e4102aad258136c770ef8979c60d4c31e0fb725eb8f04c5e7014c32349

data/.gitignore

@@ -0,0 +1,23 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp
+log/*.log
+test/dummy/db/*.sqlite3
+test/dummy/db/*.sqlite3-journal
+test/dummy/log/*.log
+test/dummy/tmp/
+test/dummy/.sass-cache

data/.travis.yml

@@ -0,0 +1,6 @@
+language: ruby
+rvm:
+  - 2.0.0
+  - 1.9.3
+notifications:
+  email: false

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in human_power.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2013 Lasse Bunk
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,91 @@
+[![Build Status](https://secure.travis-ci.org/lassebunk/human_power.png)](http://travis-ci.org/lassebunk/human_power)
+
+# Human Power
+
+Human Power lets you write your robots.txt in plain Ruby and force the robots into submission!
+
+![Bingbot getting taught](http://i.imgur.com/77WVSQH.jpg)
+
+## Installation
+
+Add this line to your *Gemfile*:
+
+    gem 'human_power'
+
+Then run:
+
+    $ bundle
+
+Or install it yourself:
+
+    $ gem install human_power
+
+If you are using Rails, you can add a sample *config/robots.rb* configuration file:
+
+    $ rails g human_power:install
+
+It will allow crawlers to access to the whole site by default.
+
+Now you can restart your server and visit `/robots.txt` to see what's generated from the new configuration file.
+
+## Usage
+
+### Using in Ruby on Rails
+
+In *config/robots.rb*:
+
+```ruby
+# Disallow everything in /admin for all user agents
+disallow_tree admin_path
+
+# Googlebot
+googlebot do
+  disallow reviews_path # Disallow a path
+  allow product_path("one") # Allow a path
+  disallow new_product_path, new_category_path # Disallow multiple paths in one line
+end
+
+# Bingbot
+bingbot do
+  disallow :all # There you go, Bingbot! (Disallow everything)
+end
+
+# Identical settings for multiple user agents
+user_agent [:bingbot, :googlebot] do
+  disallow login_path
+end
+
+# Custom user agent
+user_agent "My Custom User Agent String" do
+  disallow some_path
+end
+
+# You have access to everything from your ApplicationController
+if request.subdomains.first == "api"
+  disallow :all
+end
+```
+
+## Caveats
+
+Human Power is great for adding rules to your robots.txt.
+You should note, however, that the user agents are sorted alphabetically upon rendering.
+This is fine for most use cases, but if you add more advanced rules relying on user agent
+order, please be sure to check that robots.txt is generated into something that meets
+your needs. If you need more advanced features and rely heavily on ordering, please submit
+an [issue](https://github.com/lassebunk/human_power/issues)
+or [pull request](https://github.com/lassebunk/human_power/pulls). Thanks.
+
+## Versioning
+
+Follows [semantic versioning](http://semver.org/).
+
+## Contributing
+
+You are very welcome to contribute to this project if you have a feature that you think others can use. I'd appreciate it.
+
+1. Fork the project
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create new pull request

data/Rakefile

@@ -0,0 +1,11 @@
+require "bundler/gem_tasks"
+require "rake/testtask"
+
+Rake::TestTask.new(:test) do |t|
+  t.libs << 'lib'
+  t.libs << 'test'
+  t.pattern = 'test/**/*_test.rb'
+  t.verbose = false
+end
+
+task default: :test

data/config/initializers/human_power.rb

@@ -0,0 +1 @@
+require "human_power/rails/controller"

data/config/routes.rb

@@ -0,0 +1,3 @@
+Rails.application.routes.draw do
+  get "robots.txt" => "human_power/rails/robots#robots", format: :text, as: :robotstxt
+end

data/human_power.gemspec

@@ -0,0 +1,24 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'human_power/version'
+
+Gem::Specification.new do |s|
+  s.name          = "human_power"
+  s.version       = HumanPower::VERSION
+  s.authors       = ["Lasse Bunk"]
+  s.email         = ["lassebunk@gmail.com"]
+  s.description   = %q{Easy generation of robots.txt.}
+  s.summary       = %q{Easy generation of robots.txt. Force the robots into submission!}
+  s.homepage      = "https://github.com/lassebunk/human_power"
+  s.license       = "MIT"
+
+  s.files         = `git ls-files`.split($/)
+  s.test_files    = s.files.grep(%r{^test/})
+  s.require_paths = ["lib"]
+
+  s.add_development_dependency "bundler", "~> 1.3"
+  s.add_development_dependency "rake"
+  s.add_development_dependency "rails", "~> 4.0.1"
+  s.add_development_dependency "sqlite3"
+end

data/lib/generators/human_power/install_generator.rb

@@ -0,0 +1,12 @@
+require 'rails/generators'
+
+module HumanPower
+  class InstallGenerator < ::Rails::Generators::Base
+    source_root File.expand_path('../templates', __FILE__)
+
+    desc "Creates a sample configuration file in config/robots.rb"
+    def create_config_file
+      copy_file "robots.rb", "config/robots.rb"
+    end
+  end
+end

data/lib/generators/human_power/templates/robots.rb

@@ -0,0 +1,39 @@
+# Allow access to the whole site
+disallow :none
+
+# Examples:
+# 
+# Disallow everything:
+# 
+#   disallow :all
+#
+# Disallow everything inside /admin:
+# 
+#   disallow_tree admin_path
+# 
+# For Googlebot:
+# 
+#   googlebot do
+#     disallow login_path
+#   end
+#
+# For Bingbot:
+# 
+#   bingbot do
+#     disallow some_path
+#   end
+# 
+# Custom user agent:
+# 
+#   user_agent "My Custom User Agent" do
+#     disallow some_path
+#   end
+# 
+# Same rules for multiple user agents:
+# 
+#   user_agent [:googlebot, :bingbot, "My Custom User Agent"] do
+#     disallow some_path
+#   end
+# 
+# Please see the readme (https://github.com/lassebunk/human_power/blob/master/README.md)
+# for details on the built-in user agents/crawlers.

data/lib/human_power/generator.rb

@@ -0,0 +1,96 @@
+module HumanPower
+  class Generator
+    DISALLOW_KEYS = { all: "*", none: "" }
+
+    def initialize(context = nil, &block)
+      @context = context
+      @current_agent_string = "*"
+      instance_eval &block if block
+    end
+
+    def user_agent(key_or_string, &block)
+      old_agent_string = @current_agent_string
+      
+      [key_or_string].flatten.each do |agent|      
+        @current_agent_string = case agent
+        when String then agent
+        when Symbol then HumanPower.user_agents.fetch(agent)
+        else raise ArgumentError, "Please supply a string or symbol to `user_agent`."
+        end
+
+        instance_eval &block
+      end
+
+      @current_agent_string = old_agent_string
+    end
+
+    def add_rule(type, *paths)
+      rules[@current_agent_string].push *paths.map { |path| path.is_a?(Symbol) ? DISALLOW_KEYS.fetch(path) : path.to_s }.map { |path| HumanPower::Rule.new(type, path) }
+    end
+
+    def add_tree_rule(type, *paths)
+      add_rule type, *paths.map { |path| path.end_with?("/") ? path : "#{path}/" }
+    end
+
+    def allow(*paths)
+      add_rule :allow, *paths
+    end
+
+    def allow_tree(*paths)
+      add_tree_rule :allow, *paths
+    end
+
+    def disallow(*paths)
+      add_rule :disallow, *paths
+    end
+
+    def disallow_tree(*paths)
+      add_tree_rule :disallow, *paths
+    end
+
+    def sitemap(*sitemap_urls)
+      sitemaps.push *sitemap_urls
+    end
+
+    def rules
+      @rules ||= Hash.new { |h, k| h[k] = [] }
+    end
+
+    def sitemaps
+      @sitemaps ||= []
+    end
+
+    def context
+      @context
+    end
+
+    def render
+      sections = []
+
+      if sitemaps.any?
+        sections << sitemaps.uniq.map { |url| "Sitemap: #{url}" }.join("\n")
+      end
+      
+      agents = rules.keys.sort.map do |agent|
+        "User-agent: #{agent}\n" +
+        rules[agent].uniq.map(&:render).join("\n")
+      end
+      
+      sections.push *agents
+
+      sections.join("\n\n")
+    end
+
+    alias :to_s :render
+
+    def method_missing(method, *args, &block)
+      if block && HumanPower.user_agents.has_key?(method)
+        user_agent method, &block
+      elsif context
+        context.send method, *args, &block
+      else
+        super
+      end
+    end
+  end
+end

data/lib/human_power/rails/controller.rb

@@ -0,0 +1,14 @@
+module HumanPower
+  module Rails
+    class RobotsController < ::ApplicationController
+      def robots
+        generator = HumanPower::Generator.new(self) do
+          file = ::Rails.root.join("config", "robots.rb").to_s
+          instance_eval open(file).read, file
+        end
+
+        render text: generator.render
+      end
+    end
+  end
+end

data/lib/human_power/rails/engine.rb

@@ -0,0 +1,6 @@
+module HumanPower
+  module Rails
+    class Engine < ::Rails::Engine
+    end
+  end
+end

data/lib/human_power/rails.rb

@@ -0,0 +1 @@
+require "human_power/rails/engine"

data/lib/human_power/rule.rb

@@ -0,0 +1,20 @@
+module HumanPower
+  class Rule
+    attr_reader :type, :path
+
+    def initialize(type, path)
+      @type = type
+      @path = path
+    end
+
+    def render
+      "#{type.to_s.capitalize}: #{path}"
+    end
+
+    alias :to_s :render
+
+    def ==(other)
+      other.type == type && other.path == path
+    end
+  end
+end

data/lib/human_power/user_agents.rb

@@ -0,0 +1,7 @@
+module HumanPower
+  DEFAULT_USER_AGENTS = {
+    all: "*",
+    googlebot: "Googlebot",
+    bingbot: "Bingbot"
+  }
+end

data/lib/human_power/version.rb

@@ -0,0 +1,3 @@
+module HumanPower
+  VERSION = "0.0.1"
+end

data/lib/human_power.rb

@@ -0,0 +1,27 @@
+require "human_power/version"
+require "human_power/generator"
+require "human_power/rule"
+require "human_power/user_agents"
+require "human_power/rails" if defined?(Rails)
+
+module HumanPower
+  class << self
+    # Yields a configuration block.
+    # 
+    #   HumanPower.configure do |config|
+    #     config.base_controller = MyOtherController
+    #   end
+    def configure(&block)
+      yield self
+    end
+
+    # Registers a user agent.
+    def register_user_agent(key, user_agent_string)
+      user_agents[key] = user_agent_string
+    end
+
+    def user_agents
+      @user_agents ||= DEFAULT_USER_AGENTS
+    end
+  end
+end

data/test/dummy/README.rdoc

@@ -0,0 +1,28 @@
+== README
+
+This README would normally document whatever steps are necessary to get the
+application up and running.
+
+Things you may want to cover:
+
+* Ruby version
+
+* System dependencies
+
+* Configuration
+
+* Database creation
+
+* Database initialization
+
+* How to run the test suite
+
+* Services (job queues, cache servers, search engines, etc.)
+
+* Deployment instructions
+
+* ...
+
+
+Please feel free to use a different markup language if you do not plan to run
+<tt>rake doc:app</tt>.

data/test/dummy/Rakefile

@@ -0,0 +1,6 @@
+# Add your own tasks in files placed in lib/tasks ending in .rake,
+# for example lib/tasks/capistrano.rake, and they will automatically be available to Rake.
+
+require File.expand_path('../config/application', __FILE__)
+
+Dummy::Application.load_tasks

data/test/dummy/app/assets/javascripts/application.js

@@ -0,0 +1,13 @@
+// This is a manifest file that'll be compiled into application.js, which will include all the files
+// listed below.
+//
+// Any JavaScript/Coffee file within this directory, lib/assets/javascripts, vendor/assets/javascripts,
+// or vendor/assets/javascripts of plugins, if any, can be referenced here using a relative path.
+//
+// It's not advisable to add code directly here, but if you do, it'll appear at the bottom of the
+// compiled file.
+//
+// Read Sprockets README (https://github.com/sstephenson/sprockets#sprockets-directives) for details
+// about supported directives.
+//
+//= require_tree .

data/test/dummy/app/assets/stylesheets/application.css

@@ -0,0 +1,13 @@
+/*
+ * This is a manifest file that'll be compiled into application.css, which will include all the files
+ * listed below.
+ *
+ * Any CSS and SCSS file within this directory, lib/assets/stylesheets, vendor/assets/stylesheets,
+ * or vendor/assets/stylesheets of plugins, if any, can be referenced here using a relative path.
+ *
+ * You're free to add application-wide styles to this file and they'll appear at the top of the
+ * compiled file, but it's generally better to create a new file per style scope.
+ *
+ *= require_self
+ *= require_tree .
+ */

data/test/dummy/app/controllers/application_controller.rb

@@ -0,0 +1,5 @@
+class ApplicationController < ActionController::Base
+  # Prevent CSRF attacks by raising an exception.
+  # For APIs, you may want to use :null_session instead.
+  protect_from_forgery with: :exception
+end

data/test/dummy/app/helpers/application_helper.rb

@@ -0,0 +1,2 @@
+module ApplicationHelper
+end

data/test/dummy/app/views/layouts/application.html.erb

@@ -0,0 +1,14 @@
+<!DOCTYPE html>
+<html>
+<head>
+  <title>Dummy</title>
+  <%= stylesheet_link_tag    "application", media: "all", "data-turbolinks-track" => true %>
+  <%= javascript_include_tag "application", "data-turbolinks-track" => true %>
+  <%= csrf_meta_tags %>
+</head>
+<body>
+
+<%= yield %>
+
+</body>
+</html>

data/test/dummy/bin/bundle

@@ -0,0 +1,3 @@
+#!/usr/bin/env ruby
+ENV['BUNDLE_GEMFILE'] ||= File.expand_path('../../Gemfile', __FILE__)
+load Gem.bin_path('bundler', 'bundle')

data/test/dummy/bin/rails

@@ -0,0 +1,4 @@
+#!/usr/bin/env ruby
+APP_PATH = File.expand_path('../../config/application',  __FILE__)
+require_relative '../config/boot'
+require 'rails/commands'

data/test/dummy/bin/rake

@@ -0,0 +1,4 @@
+#!/usr/bin/env ruby
+require_relative '../config/boot'
+require 'rake'
+Rake.application.run

data/test/dummy/config/application.rb

@@ -0,0 +1,23 @@
+require File.expand_path('../boot', __FILE__)
+
+require 'rails/all'
+
+Bundler.require(*Rails.groups)
+require "human_power"
+
+module Dummy
+  class Application < Rails::Application
+    # Settings in config/environments/* take precedence over those specified here.
+    # Application configuration should go into files in config/initializers
+    # -- all .rb files in that directory are automatically loaded.
+
+    # Set Time.zone default to the specified zone and make Active Record auto-convert to this zone.
+    # Run "rake -D time" for a list of tasks for finding time zone names. Default is UTC.
+    # config.time_zone = 'Central Time (US & Canada)'
+
+    # The default locale is :en and all translations from config/locales/*.rb,yml are auto loaded.
+    # config.i18n.load_path += Dir[Rails.root.join('my', 'locales', '*.{rb,yml}').to_s]
+    # config.i18n.default_locale = :de
+  end
+end
+

data/test/dummy/config/boot.rb

@@ -0,0 +1,5 @@
+# Set up gems listed in the Gemfile.
+ENV['BUNDLE_GEMFILE'] ||= File.expand_path('../../../../Gemfile', __FILE__)
+
+require 'bundler/setup' if File.exists?(ENV['BUNDLE_GEMFILE'])
+$LOAD_PATH.unshift File.expand_path('../../../../lib', __FILE__)

data/test/dummy/config/database.yml

@@ -0,0 +1,25 @@
+# SQLite version 3.x
+#   gem install sqlite3
+#
+#   Ensure the SQLite 3 gem is defined in your Gemfile
+#   gem 'sqlite3'
+development:
+  adapter: sqlite3
+  database: db/development.sqlite3
+  pool: 5
+  timeout: 5000
+
+# Warning: The database defined as "test" will be erased and
+# re-generated from your development database when you run "rake".
+# Do not set this db to the same as development or production.
+test:
+  adapter: sqlite3
+  database: db/test.sqlite3
+  pool: 5
+  timeout: 5000
+
+production:
+  adapter: sqlite3
+  database: db/production.sqlite3
+  pool: 5
+  timeout: 5000

data/test/dummy/config/environment.rb

@@ -0,0 +1,5 @@
+# Load the Rails application.
+require File.expand_path('../application', __FILE__)
+
+# Initialize the Rails application.
+Dummy::Application.initialize!

data/test/dummy/config/environments/development.rb

@@ -0,0 +1,29 @@
+Dummy::Application.configure do
+  # Settings specified here will take precedence over those in config/application.rb.
+
+  # In the development environment your application's code is reloaded on
+  # every request. This slows down response time but is perfect for development
+  # since you don't have to restart the web server when you make code changes.
+  config.cache_classes = false
+
+  # Do not eager load code on boot.
+  config.eager_load = false
+
+  # Show full error reports and disable caching.
+  config.consider_all_requests_local       = true
+  config.action_controller.perform_caching = false
+
+  # Don't care if the mailer can't send.
+  config.action_mailer.raise_delivery_errors = false
+
+  # Print deprecation notices to the Rails logger.
+  config.active_support.deprecation = :log
+
+  # Raise an error on page load if there are pending migrations
+  config.active_record.migration_error = :page_load
+
+  # Debug mode disables concatenation and preprocessing of assets.
+  # This option may cause significant delays in view rendering with a large
+  # number of complex assets.
+  config.assets.debug = true
+end