probot 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 65dd2e16a696a2500937d32ae744262f74be414620e5505838fde49b8847d34d
+  data.tar.gz: 5f50733228c4c4eec37218eda00ceb4c2dd7eca801df6dd7345dd9e7edc94516
+SHA512:
+  metadata.gz: 33a44b9aba61643781e697e1b0fd54ac8d1afb40a61b1e2dc13d174aeea1b4ec1e6c2b762d122a2e1f5e50b447881b28c57cedaaff9a05f1174ce0d531c7f605
+  data.tar.gz: 85615bf3573de8af826308f5bc36df231581de0da4346081a44632cd5882df8c7574ce9889f586dc7fdde2c881a6da13edf49b806bc9f76d0676b20cd516a789

data/.standard.yml

@@ -0,0 +1,3 @@
+# For available configuration options, see:
+#   https://github.com/testdouble/standard
+ruby_version: 3.0.0

data/CHANGELOG.md

@@ -0,0 +1,5 @@
+## [Unreleased]
+
+## [0.1.0] - 2023-09-09
+
+- Initial release

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2023 Dan Milne
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,93 @@
+# Probot
+
+OMG another Ruby Robot.txt parser? It was an accident, I didn't mean to make it and I shouldn't have but here we are. It started out tiny and grew. Yes I should have used one of the other gems.
+
+Does this even deserve a gem? Feel free to just copy and paste the single file which implements this - one less dependency eh? 
+
+On the plus side of this yak shaving, there are some nice features I don't think the others have.
+
+1. Support for consecutive user agents making up a single record:
+
+```txt
+User-agent: first-agent
+User-agent: second-agent
+Disallow: /
+```
+
+This record blocks both first-agent and second-agent from the site.
+
+2. It selects the most specific allow / disallow rule, using rule length as a proxy for specificity. You can also ask it to show you the matching rules and their scores. 
+
+```ruby
+txt = %Q{
+User-agent: *
+Disallow: /dir1
+Allow: /dir1/dir2
+Disallow: /dir1/dir2/dir3
+}
+Probot.new(txt).matches("/dir1/dir2/dir3")
+=> {:disallowed=>{/\/dir1/=>5, /\/dir1\/dir2\/dir3/=>15}, :allowed=>{/\/dir1\/dir2/=>10}}
+```
+
+In this case, we can see the Disallow rule with length 15 would be followed.
+
+3. It sets the User-Agent string when fetching robots.txt
+
+## Installation
+
+Install the gem and add to the application's Gemfile by executing:
+
+    $ bundle add probot
+
+If bundler is not being used to manage dependencies, install the gem by executing:
+
+    $ gem install probot
+
+## Usage
+
+It's straightforward to use. Instantiate it if you'll make a few requests:
+
+```ruby
+> r = Probot.new('https://booko.info', agent: 'BookScraper')
+> r.rules
+=>  {"*"=>{"disallow"=>[/\/search/, /\/products\/search/, /\/.*\/refresh_prices/, /\/.*\/add_to_cart/, /\/.*\/get_prices/, /\/lists\/add/, /\/.*\/add$/, /\/api\//, /\/users\/bits/, /\/users\/create/, /\/prices\//, /\/widgets\/issue/], "allow"=>[], "crawl_delay"=>0, "crawl-delay"=>0.1},
+ "YandexBot"=>{"disallow"=>[], "allow"=>[], "crawl_delay"=>0, "crawl-delay"=>300.0}}
+
+> r.allowed?("/abc/refresh_prices")
+=> false
+> r.allowed?("https://booko.info/9780765397522/All-Systems-Red")
+=> true
+> r.allowed?("https://booko.info/9780765397522/refresh_prices")
+=> false
+```
+
+Or just one-shot it for one-offs: 
+
+```ruby
+Probot.allowed?("https://booko.info/9780765397522/All-Systems-Red", agent: "BookScraper")
+```
+
+
+## Development
+
+After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake test` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
+
+To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and the created tag, and push the `.gem` file to [rubygems.org](https://rubygems.org).
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at https://github.com/[USERNAME]/Probot.
+
+## Further Reading
+
+*  https://moz.com/learn/seo/robotstxt
+*  https://stackoverflow.com/questions/45293419/order-of-directives-in-robots-txt-do-they-overwrite-each-other-or-complement-ea
+*  https://developers.google.com/search/docs/crawling-indexing/robots/robots_txt
+*  https://developers.google.com/search/docs/crawling-indexing/robots/create-robots-txt
+
+*  https://github.com/google/robotstxt  - Google's official parser
+
+
+## License
+
+The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT).

data/Rakefile

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+require "bundler/gem_tasks"
+require "rake/testtask"
+
+Rake::TestTask.new(:test) do |t|
+  t.libs << "test"
+  t.libs << "lib"
+  t.test_files = FileList["test/**/test_*.rb"]
+end
+
+require "standard/rake"
+
+task default: %i[test standard]

data/lib/probot/version.rb

@@ -0,0 +1,3 @@
+class Probot
+  VERSION = "0.1.0"
+end

data/lib/probot.rb

@@ -0,0 +1,157 @@
+# frozen_string_literal: true
+
+require "uri"
+require "net/http"
+
+# https://moz.com/learn/seo/robotstxt
+# https://stackoverflow.com/questions/45293419/order-of-directives-in-robots-txt-do-they-overwrite-each-other-or-complement-ea
+# https://developers.google.com/search/docs/crawling-indexing/robots/create-robots-txt
+# https://developers.google.com/search/docs/crawling-indexing/robots/robots_txt
+#
+# https://github.com/google/robotstxt  - Google's official parser
+
+# Note: User-agent found on consecutive lines are considered to be part of the same record.
+# Note: Google ignores crawl_delay
+# Note: Google does not consider crawl_delay or sitemap to be part of the per-agent records.
+
+# Two main parts of this class:
+#   Parse a robots.txt file
+#   Find the most specific rule for a given URL. We use the length of the regexp as a proxy for specificity.
+
+class Probot
+  attr_reader :rules, :sitemap, :doc
+  attr_accessor :agent
+
+  def initialize(data, agent: "*")
+    raise ArgumentError, "The first argument must be a string" unless data.is_a?(String)
+    @agent = agent
+
+    @rules = {}
+    @current_agents = ["*"]
+    @current_agents.each { |agent| @rules[agent] ||= {"disallow" => [], "allow" => [], "crawl_delay" => 0} }
+    @sitemaps = []
+
+    @doc = data.start_with?("http") ? fetch_robots_txt(data) : data
+    parse(@doc)
+  end
+
+  def request_headers = (agent == "*") ? {} : {"User-Agent" => @agent}
+
+  def fetch_robots_txt(url)
+    Net::HTTP.get(URI(url).tap { |u| u.path = "/robots.txt" }, request_headers)
+  rescue
+    ""
+  end
+
+  def crawl_delay = rules.dig(@agent, "crawl_delay")
+
+  def found_agents = rules.keys
+
+  def disallowed = rules.dig(@agent, "disallow") || rules.dig("*", "disallow")
+
+  def allowed = rules.dig(@agent, "allow") || rules.dig("*", "allow")
+
+  def disallowed_matches(url) = disallowed.select { |disallowed_url| url.match?(disallowed_url) }.to_h { |rule| [rule, pattern_length(rule)] }
+
+  def allowed_matches(url) = allowed.select { |allowed_url| url.match?(allowed_url) }.to_h { |rule| [rule, pattern_length(rule)] }
+
+  def matches(url) = {disallowed: disallowed_matches(url), allowed: allowed_matches(url)}
+
+  def disallowed_best(url) = disallowed_matches(url).max_by { |k, v| v }
+
+  def allowed_best(url) = allowed_matches(url).max_by { |k, v| v }
+
+  def matching_rule(url) = (disallowed_best(url)&.last.to_i > allowed_best(url)&.last.to_i) ? {disallow: disallowed_best(url)&.first} : {allow: allowed_best(url)&.first}
+
+  # If a URL is not disallowed, it is allowed - so we check if it is explictly disallowed and if not, it's allowed.
+  def allowed?(url) = !disallowed?(url)
+
+  def disallowed?(url) = matching_rule(url)&.keys&.first == :disallow
+
+  def parse(doc)
+    # We need to handle consective user-agent lines, which are considered to be part of the same record.
+    subsequent_agent = false
+
+    doc.lines.each do |line|
+      next if line.start_with?("#") || !line.include?(":") || line.split(":").length < 2
+
+      data = ParsedLine.new(line)
+
+      if data.agent?
+        if subsequent_agent
+          @current_agents << data.value
+        else
+          @current_agents = [data.value]
+          subsequent_agent = true
+        end
+
+        @current_agents.each { |agent| rules[agent] ||= {"disallow" => [], "allow" => [], "crawl_delay" => 0} }
+        next
+      end
+
+      # All Regex characters are escaped, then we unescape * and $ as they may used in robots.txt
+
+      if data.allow? || data.disallow?
+        @current_agents.each { |agent| rules[agent][data.key] << Regexp.new(Regexp.escape(data.value).gsub('\*', ".*").gsub('\$', "$")) }
+
+        subsequent_agent = false # When user-agent strings are found on consecutive lines, they are considered to be part of the same record. Google ignores crawl_delay.
+        next
+      end
+
+      if data.crawl_delay?
+        @current_agents.each { |agent| rules[agent][data.key] = data.value }
+        next
+      end
+
+      if data.sitemap?
+        @sitemap = URI(data.value).path
+        next
+      end
+
+      @current_agents.each { |agent| rules[agent][data.key] = data.value }
+    end
+  end
+
+  def pattern_length(regexp) = regexp.source.gsub(/(\\[\*\$\.])/, "*").length
+
+  # ParedLine Note: In the case of 'Sitemap: https://example.com/sitemap.xml', raw_value needs to rejoin after splitting the URL.
+
+  ParsedLine = Struct.new(:input_string) do
+    def key = input_string.split(":").first&.strip&.downcase
+
+    def raw_value = input_string.split(":").slice(1..)&.join(":")&.strip
+
+    def clean_value = raw_value.split("#").first&.strip
+
+    def agent? = key == "user-agent"
+
+    def disallow? = key == "disallow"
+
+    def allow? = key == "allow"
+
+    def crawl_delay? = key == "crawl-delay"
+
+    def sitemap? = key == "sitemap"
+
+    def value
+      return clean_value.to_f if crawl_delay?
+      return URI(clean_value).to_s if disallow? || allow?
+
+      raw_value
+    rescue URI::InvalidURIError
+      raw_value
+    end
+  end
+
+  def self.allowed?(url, agent: "*") = Probot.new(url, agent: agent).allowed?(url)
+end
+
+# Probot.allowed?("https://booko.info/9780765397522/All-Systems-Red")
+# => true
+# r = Probot.new('https://booko.info', agent: 'YandexBot')
+# r = Probot.new('https://www.allenandunwin.com')
+# $ Probot.new('https://www.amazon.com/').matches("/gp/wishlist/ipad-install/gcrnsts")
+# => {:disallowed=>{/\/wishlist\//=>10, /\/gp\/wishlist\//=>13, /.*\/gcrnsts/=>10}, :allowed=>{/\/gp\/wishlist\/ipad\-install.*/=>28}}
+#
+# Test with
+# assert Probot.new(nil, doc: %Q{allow: /$\ndisallow: /}).matching_rule('https://example.com/page.htm') == {disallow: /\//}

data/probot.gemspec

@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+require_relative "lib/probot/version"
+
+Gem::Specification.new do |spec|
+  spec.name = "probot"
+  spec.version = Probot::VERSION
+  spec.authors = ["Dan Milne"]
+  spec.email = ["d@nmilne.com"]
+
+  spec.summary = "A robots.txt parser."
+  spec.description = "A fully featured robots.txt parser."
+  spec.homepage = "http://github.com/dkam/probot"
+  spec.license = "MIT"
+  spec.required_ruby_version = ">= 3.0"
+
+  spec.metadata["homepage_uri"] = spec.homepage
+  spec.metadata["source_code_uri"] = "http://github.com/dkam/probot"
+  spec.metadata["changelog_uri"] = "http://github.com/dkam/probot/CHANGELOG.md"
+
+  # Specify which files should be added to the gem when it is released.
+  # The `git ls-files -z` loads the files in the RubyGem that have been added into git.
+  spec.files = Dir.chdir(__dir__) do
+    `git ls-files -z`.split("\x0").reject do |f|
+      (File.expand_path(f) == __FILE__) ||
+        f.start_with?(*%w[bin/ test/ spec/ features/ .git .circleci appveyor Gemfile])
+    end
+  end
+  spec.bindir = "exe"
+  spec.executables = spec.files.grep(%r{\Aexe/}) { |f| File.basename(f) }
+  spec.require_paths = ["lib"]
+
+  # Uncomment to register a new dependency of your gem
+  # spec.add_dependency "example-gem", "~> 1.0"
+
+  # For more information and examples about making a new gem, check out our
+  # guide at: https://bundler.io/guides/creating_gem.html
+end

data/sig/probot.rbs

@@ -0,0 +1,4 @@
+module Probot
+  VERSION: String
+  # See the writing guide of rbs: https://github.com/ruby/rbs#guides
+end

metadata

@@ -0,0 +1,55 @@
+--- !ruby/object:Gem::Specification
+name: probot
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Dan Milne
+autorequire:
+bindir: exe
+cert_chain: []
+date: 2023-09-10 00:00:00.000000000 Z
+dependencies: []
+description: A fully featured robots.txt parser.
+email:
+- d@nmilne.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".standard.yml"
+- CHANGELOG.md
+- LICENSE.txt
+- README.md
+- Rakefile
+- lib/probot.rb
+- lib/probot/version.rb
+- probot.gemspec
+- sig/probot.rbs
+homepage: http://github.com/dkam/probot
+licenses:
+- MIT
+metadata:
+  homepage_uri: http://github.com/dkam/probot
+  source_code_uri: http://github.com/dkam/probot
+  changelog_uri: http://github.com/dkam/probot/CHANGELOG.md
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '3.0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.4.19
+signing_key:
+specification_version: 4
+summary: A robots.txt parser.
+test_files: []