gort 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: e05f323b6f9699a6f39f7042d494e18856bd954b974cf6943d0a0fa6bdb1f263
+  data.tar.gz: 396813a5d7c484e5408603c17bb76d79ec4debce48edec8a6b8ecf8eeb0231da
+SHA512:
+  metadata.gz: 9978aba28e9e77ea750cf61de9a22137706fc578fbe0bfbc348d9c97b4d516ea3b6a2e11250f115a2eeae8fea9752c48dc5803556fd3e9a4c0dcfc1e4e9a02e0
+  data.tar.gz: cae8fc644d21a8d31da01b2390f801ceb06b0fc857d1c3d900a62496ff111f41636e09316c86520ab829add7248dcfb1d2a818eea0dd04b030429f8734388be8

checksums.yaml.gz.sig

@@ -0,0 +1,2 @@
+o�@(�{@�B�>I��(�S�S�AĤE*a�i��B�M��܄}C;�G�@�M�0��^�x!���r���;��7�܄�闶������_�)��w�Ⱦ�a�ŴR�:�{�Y@r���˾��m>
�Ի{"~��O��@I+�#S
+�r��8p���B�5�h�%��-(�c�N��ϼDn3�} �\I�.~�}���2��I���<�6��ʻ�V�2K��&�[+���^|V6Ƹ���ͼ�

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2024 Alexander Mankuta
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/lib/gort/allow_rule.rb

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+require_relative "path_rule"
+
+module Gort
+  # Allow rule.
+  class AllowRule < PathRule
+    # @param value [String] the path pattern to allow.
+    def initialize(value)
+      super(:allow, value)
+    end
+  end
+end

data/lib/gort/disallow_rule.rb

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+require_relative "path_rule"
+
+module Gort
+  # Disallow rule.
+  class DisallowRule < PathRule
+    # @param value [String] the path pattern to disallow.
+    def initialize(value)
+      super(:disallow, value)
+    end
+  end
+end

data/lib/gort/group.rb

@@ -0,0 +1,85 @@
+# frozen_string_literal: true
+
+require_relative "rule_set"
+
+module Gort
+  # An access group
+  class Group < RuleSet
+    # Is this group valid?
+    #
+    # A valid group has at least one valid user-agent rule.
+    #
+    # @return [Boolean]
+    # @see UserAgentRule#valid?
+    def valid?
+      @valid ||=
+        rules.any? { |rule| rule.is_a?(UserAgentRule) && rule.valid? }
+    end
+
+    # Does this group apply to this specific user agent?
+    #
+    # This performa user agent matcchign acording to the RFC.
+    #
+    # @param user_agent [String]
+    # @return [Boolean]
+    def apply?(user_agent)
+      apply_to_all? || user_agent.match?(user_agent_regexp)
+    end
+
+    # @!group Formatting Methods
+
+    # A human readable representation of the group.
+    #
+    # @return [String]
+    # @tool
+    #   :nocov:
+    def inspect
+      "#<#{self.class.name}:#{object_id} #{rules.inspect}>"
+    end
+    # :nocov:
+
+    # Produces a pretty human readable representation of the group.
+    #
+    # @param pp [PrettyPrint] pretty printer
+    # @return [void]
+    # @tool
+    #   :nocov:
+    def pretty_print(pp)
+      pp.text("#{self.class.name}/#{object_id}")
+      pp.group(1, "[", "]") do
+        pp.breakable("")
+        pp.seplist(rules) do |rule|
+          pp.pp(rule)
+        end
+        pp.breakable("")
+      end
+    end
+    # :nocov:
+
+    # @!endgroup Formatting Methods
+
+    private
+
+    # Does this rule apply to all user agents?
+    #
+    # Effectively, is this rule is a `*` rule.
+    #
+    # @return [Boolean]
+    def apply_to_all?
+      @apply_to_all ||= rules.any? { |rule| rule.is_a?(UserAgentRule) && rule.valid? && rule.value == "*" }
+    end
+
+    # A compiled Regexp that mathes all user agents in this group.
+    #
+    # @return [Regexp]
+    def user_agent_regexp
+      @user_agent_regexp ||=
+        begin
+          specific_user_agent_rules = rules.select { |rule|
+            rule.is_a?(UserAgentRule) && rule.valid? && (rule.value != "*")
+          }
+          Regexp.new(specific_user_agent_rules.map { Regexp.escape(_1.value) }.join("|"), Regexp::IGNORECASE)
+        end
+    end
+  end
+end

data/lib/gort/invalid_line.rb

@@ -0,0 +1,45 @@
+# frozen_string_literal: true
+
+module Gort
+  # Represents an invalid line in a robots.txt file.
+  #
+  # @note Technically, the RFC doesn't have invalid lines in its grammar
+  #   but there are just too many broken robots.txt files on the internet.
+  #
+  # An invalid line is a line that can not be parsed as a rule and is not a comment.
+  class InvalidLine
+    # @param text [String] content of the line
+    def initialize(text)
+      @value = text
+    end
+
+    # Content of the line.
+    # @return [String]
+    attr_reader :value
+
+    # @!group Formatting Methods
+
+    # A human readable representation of the invalid line.
+    #
+    # @return [String]
+    # @tool
+    #   :nocov:
+    def inspect
+      %(#<#{self.class.name}:#{object_id} "#{value}">)
+    end
+    # :nocov:
+
+    # Produces a pretty human readable representation of the invalid line.
+    #
+    # @param pp [PrettyPrint] pretty printer
+    # @return [void]
+    # @tool
+    #   :nocov:
+    def pretty_print(pp)
+      pp.text("#{self.class.name}/#{object_id}< #{value} >")
+    end
+    # :nocov:
+
+    # @!endgroup Formatting Methods
+  end
+end

data/lib/gort/parser.rb

@@ -0,0 +1,144 @@
+# frozen_string_literal: true
+
+require_relative "invalid_line"
+require_relative "rule"
+require_relative "user_agent_rule"
+require_relative "allow_rule"
+require_relative "disallow_rule"
+require_relative "rule_set"
+require_relative "group"
+require_relative "robots_txt"
+
+module Gort
+  # robots.txt parser. It implements the parsing logic according to RFC 9309, including errata.
+  class Parser
+    # You may get this error if the input does not look like a text file.
+    class BinaryInputError < Error; end
+
+    # You may get this error if the input looks like a text file but its encoding is invalid.
+    class InvalidEncodingError < Error; end
+
+    UTF_8_BOM = "\ufeff"
+    private_constant :UTF_8_BOM
+
+    # @param input [String] The robots.txt content to parse. It must be encoded in UTF-8 or compatible encoding.
+    def initialize(input)
+      @input = detect_and_fix_encoding(input).then { |string| strip_bom(string) }
+    end
+
+    # RFC does not explicitly define the generic rule name syntax. It only defines that it has to be case-insensitive.
+    # It also provides a few pre-defined rule names such as User-Agent, Allow, and Disallow.
+    # Things that might be different from the RFC intention:
+    # - The rule name must start with a letter. RFC might allow other characters.
+    # - The rule name might contain underscores. RFC doesn't mention underscores.
+    # - The rule name might contain digits. RFC doesn't mention digits, either.
+    #
+    # This is only used for plausible rule detection.
+    RULE_KEY = /\A[a-z][a-z0-9_-]*\s*:/i
+    private_constant :RULE_KEY
+
+    # Actually parse the file.
+    #
+    # @return [Gort::RobotsTxt]
+    def parse
+      content_lines =
+        input.lines.map { |line|
+          line.split("#", 2).first.strip
+        }
+        .reject(&:empty?)
+
+      rules = content_lines.map { |line| parse_line(line) }
+      grouped_rules, standalone_rules = partition_rules(rules)
+      groups = group_rules(grouped_rules)
+
+      RobotsTxt.new(groups + standalone_rules)
+    end
+
+    private
+
+    # @return [String]
+    attr_reader :input
+
+    # @param string [String]
+    # @return [String]
+    def detect_and_fix_encoding(string)
+      string.encode(Encoding::UTF_8)
+    rescue EncodingError
+      require "rchardet"
+      result = CharDet.detect(string)
+      raise BinaryInputError, "Input does not look like text" if result["encoding"].nil? || result["confidence"] < 0.25
+
+      begin
+        string
+          .dup
+          .force_encoding(result["encoding"])
+          .encode(Encoding::UTF_8)
+      rescue EncodingError
+        raise InvalidEncodingError, "Input string looks like text but its encoding is invalid."
+      end
+    end
+
+    # @param string [String]
+    # @return [String]
+    def strip_bom(string)
+      if string[0] == UTF_8_BOM
+        string[1..] # Remove BOM
+      else
+        string
+      end
+    end
+
+    # @param line [String]
+    # @return [UserAgentRule, AllowRule, DisallowRule, Rule, InvalidLine]
+    def parse_line(line)
+      if line.match?(RULE_KEY)
+        # @type var key: String
+        # @type var value: String
+        key, value = line.split(":", 2).map(&:strip)
+        case key.downcase
+        when "user-agent"
+          UserAgentRule.new(value)
+        when "allow"
+          AllowRule.new(value)
+        when "disallow"
+          DisallowRule.new(value)
+        else
+          Rule.new(key, value)
+        end
+      else
+        InvalidLine.new(line)
+      end
+    end
+
+    # @param rules [Array<UserAgentRule, AllowRule, DisallowRule, Rule, InvalidLine>]
+    # @return [(Array<UserAgentRule, AllowRule, DisallowRule>, Array<AllowRule, DisallowRule, Rule, InvalidLine>)]
+    def partition_rules(rules)
+      standalone_rules = []
+      grouped_rules = []
+      rules.each do |rule|
+        case rule
+        when UserAgentRule
+          grouped_rules << rule
+        when AllowRule, DisallowRule
+          if grouped_rules.empty?
+            standalone_rules << rule
+          else
+            grouped_rules << rule
+          end
+        else
+          standalone_rules << rule
+        end
+      end
+
+      [grouped_rules, standalone_rules]
+    end
+
+    # @param rules [Array<UserAgentRule, AllowRule, DisallowRule>]
+    # @return [Array<Group>]
+    def group_rules(rules)
+      rules
+        .slice_when { |a, b| !a.is_a?(UserAgentRule) && b.is_a?(UserAgentRule) }
+        .map { |group| Group.new(group) }
+    end
+  end
+end

data/lib/gort/path_rule.rb

@@ -0,0 +1,98 @@
+# frozen_string_literal: true
+
+require_relative "rule"
+require "addressable/uri"
+
+module Gort
+  # A rule that matches a path and query string.
+  #
+  # @abstract
+  class PathRule < Rule
+    # Path patter has to start with a slash and not contain control characters or hash.
+    # It also has to be a valid UTF-8 string but this is checked diring parsing.
+    # It also can be empty.
+    PATH_PATTERN = %r{\A(?:[/*][^\u0000-\u0020\u0023$]*\$?)?\z}u
+    private_constant :PATH_PATTERN
+
+    def valid?
+      value.match?(PATH_PATTERN)
+    end
+
+    # Match the path and query string against the rule.
+    # Invalid rules never match.
+    # Empty rules never match, either. This is not explicitly stated in the RFC
+    # but it is explicitly described in previous robots.txt documents.
+    #
+    # @param path_and_query [String]
+    # @return [nil, (Integer, PathRule)]
+    #   - +nil+ if the rule does not match the path and query string.
+    #   - An array with the number of bytes matched and the rule itself if the rule matches.
+    def match(path_and_query)
+      return nil if !valid? || value.empty?
+
+      path_and_query = normalize_path_and_query(path_and_query)
+      match = path_and_query.match(regexp)
+      return nil unless match
+
+      [match.to_s.bytesize, self]
+    end
+
+    # @!group Formatting Methods
+
+    # A human readable representation of the rule.
+    #
+    # @return [String]
+    # @tool
+    #   :nocov:
+    def inspect
+      %(#<#{self.class.name}:#{object_id} "#{value}">)
+    end
+    # :nocov:
+
+    # Produces a pretty human readable representation of the rule.
+    #
+    # @param pp [PrettyPrint] pretty printer
+    # @return [void]
+    # @tool
+    #   :nocov:
+    def pretty_print(pp)
+      pp.text("#{self.class.name}/#{object_id}< #{value} >")
+    end
+    # :nocov:
+
+    # @!endgroup Formatting Methods
+
+    private
+
+    # @param path_and_query [String]
+    # @return [String]
+    def normalize_path_and_query(path_and_query)
+      pq = Addressable::URI.parse(path_and_query).normalize
+      pq.scheme = nil
+      pq.authority = nil
+      pq.fragment = nil
+      pq.to_s
+    end
+
+    # @return [Regexp]
+    def regexp
+      @regexp ||=
+        begin
+          parts = value.scan(/[^*$]+|[*$]/)
+          regexp_parts =
+            parts.map { |part|
+              case part
+              when "*"
+                ".*"
+              when "$"
+                "\\z"
+              else
+                Regexp.escape(Addressable::URI.normalized_encode(part))
+              end
+            }
+
+          Regexp.new("\\A#{regexp_parts.join}")
+        end
+    end
+  end
+end

data/lib/gort/robots_txt.rb

@@ -0,0 +1,96 @@
+# frozen_string_literal: true
+
+module Gort
+  # Represents a robots.txt file.
+  class RobotsTxt
+    ROBOTS_TXT_PATH = "/robots.txt"
+    private_constant :ROBOTS_TXT_PATH
+
+    def initialize(rules)
+      @rules = rules
+    end
+
+    # @return [Array<Rule, Group, InvalidLine>]
+    attr_reader :rules
+
+    # Is this path allowed for the given user agent?
+    #
+    # @param user_agent [String]
+    # @param path_and_query [String]
+    # @return [Boolean]
+    # @see PathRule#match
+    # @see #disallow?
+    def allow?(user_agent, path_and_query)
+      return true if path_and_query == ROBOTS_TXT_PATH
+
+      top_match =
+        matches(user_agent, path_and_query)
+        .compact
+        # This is an arcane bit.
+        # The rules are reverse sorted by match length (i.e. longest first),
+        # and then by class name using the fact that allow goes before disallow.
+        # This is the rule precedence order defined in the RFC.
+        .min_by { |(match_length, rule)| [-match_length, rule.class.name] }
+
+      # Allow if there is no match or the top match is an allow rule.
+      top_match.nil? || top_match.last.is_a?(AllowRule)
+    end
+
+    # Is this path disallowed for the given user agent?
+    #
+    # @param user_agent [String]
+    # @param path_and_query [String]
+    # @return [Boolean]
+    # @see PathRule#match
+    # @see #allow?
+    def disallow?(user_agent, path_and_query)
+      !allow?(user_agent, path_and_query)
+    end
+
+    # @!group Formatting Methods
+
+    # A human readable representation of the robots.txt.
+    #
+    # @return [String]
+    # @tool
+    #   :nocov:
+    def inspect
+      "#<#{self.class.name}:#{object_id} #{rules.inspect}>"
+    end
+    # :nocov:
+
+    # Produces a pretty human readable representation of the robots.txt.
+    #
+    # @param pp [PrettyPrint] pretty printer
+    # @return [void]
+    # @tool
+    #   :nocov:
+    def pretty_print(pp)
+      pp.text("#{self.class.name}/#{object_id}")
+      pp.group(1, "[", "]") do
+        pp.breakable("")
+        pp.seplist(rules) do |rule|
+          pp.pp(rule)
+        end
+        pp.breakable("")
+      end
+    end
+    # :nocov:
+
+    # @!endgroup Formatting Methods
+
+    private
+
+    def matches(user_agent, path)
+      # @type var groups: Array<Group>
+      groups = rules.select { |rule| rule.is_a?(Group) && rule.valid? && rule.apply?(user_agent) }
+      groups.flat_map do |group|
+        group.rules.filter_map do |rule|
+          next unless rule.is_a?(PathRule)
+
+          rule.match(path)
+        end
+      end
+    end
+  end
+end

data/lib/gort/rule.rb

@@ -0,0 +1,48 @@
+# frozen_string_literal: true
+
+module Gort
+  # Generic rule.
+  # This represents an entry that looks like a valid rule but otherwise doesn't
+  # have a more specialized implementation.
+  class Rule
+    # @param name [Symbol] The name of the rule.
+    # @param value [String] The value of the rule.
+    def initialize(name, value)
+      @name = name.downcase.to_sym
+      @value = value
+    end
+
+    # The name of the rule.
+    # @return [Symbol]
+    attr_reader :name
+
+    # The value of the rule.
+    # @return [String]
+    attr_reader :value
+
+    # @!group Formatting Methods
+
+    # A human readable representation of the rule.
+    #
+    # @return [String]
+    # @tool
+    #   :nocov:
+    def inspect
+      %(#<#{self.class.name}:#{object_id} "#{name}", "#{value}">)
+    end
+    # :nocov:
+
+    # Produces a pretty human readable representation of the rule.
+    #
+    # @param pp [PrettyPrint] pretty printer
+    # @return [void]
+    # @tool
+    #   :nocov:
+    def pretty_print(pp)
+      pp.text("#{self.class.name}/#{object_id}< #{name.inspect}, #{value} >")
+    end
+    # :nocov:
+
+    # @!endgroup Formatting Methods
+  end
+end

data/lib/gort/rule_set.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+module Gort
+  # Abstract rule set.
+  #
+  # @abstract
+  class RuleSet
+    # @param rules [Array<Group, UserAgentRule, AllowRule, DisallowRule, Rule, InvalidLine>]
+    #    The rules. Or invalid lines.
+    def initialize(*rules)
+      @rules = rules.flatten.freeze
+    end
+
+    # Rules in this set
+    # @return [Array<Group, UserAgentRule, AllowRule, DisallowRule, Rule, InvalidLine>]
+    attr_reader :rules
+
+    # Make a new set by mergin this one with another.
+    # @param other [RuleSet]
+    # @return [RuleSet]
+    def merge(other)
+      self.class.new(rules + other.rules)
+    end
+  end
+end

data/lib/gort/user_agent_rule.rb

@@ -0,0 +1,52 @@
+# frozen_string_literal: true
+
+require_relative "rule"
+
+module Gort
+  # User-agent rule.
+  class UserAgentRule < Rule
+    def initialize(value)
+      super(:"user-agent", value)
+    end
+
+    PRODUCT_TOKEN_RE = /\A([a-z_-]+|\*)\z/i
+    private_constant :PRODUCT_TOKEN_RE
+
+    # Returns +true+ if the value is a valid user agent.
+    #
+    # A user agent token is a sequence of letters (a—z, A—Z), digits (0—9),
+    # underscores (_), or hyphens (-). Alternatively, a single asterisk (*) is also allowed.
+    #
+    # @return [Boolean]
+    #   - +true+ if the value is a valid product token
+    #   - +false+ otherwise
+    def valid?
+      value.match?(PRODUCT_TOKEN_RE)
+    end
+
+    # @!group Formatting Methods
+
+    # A human readable representation of the rule.
+    #
+    # @return [String]
+    # @tool
+    #   :nocov:
+    def inspect
+      %(#<#{self.class.name}:#{object_id} "#{value}">)
+    end
+    # :nocov:
+
+    # Produces a pretty human readable representation of the rule.
+    #
+    # @param pp [PrettyPrint] pretty printer
+    # @return [void]
+    # @tool
+    #   :nocov:
+    def pretty_print(pp)
+      pp.text("#{self.class.name}/#{object_id}< #{value} >")
+    end
+    # :nocov:
+
+    # @!endgroup Formatting Methods
+  end
+end

data/lib/gort/version.rb

@@ -0,0 +1,6 @@
+# frozen_string_literal: true
+
+module Gort
+  # Gem version
+  VERSION = "0.1.0"
+end

data/lib/gort.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+require_relative "gort/version"
+
+# Gort is a robots.txt parser and evaluator.
+module Gort
+  # Gort's top error class. All other errors inherit from this.
+  class Error < StandardError; end
+
+  # Parse the given robots.txt input and return a RobotsTxt instance.
+  #
+  # @param input [String] the robots.txt input to parse
+  # @return [RobotsTxt] the parsed robots.txt
+  def self.parse(input)
+    Parser.new(input).parse
+  end
+end
+
+require_relative "gort/parser"

metadata

@@ -0,0 +1,105 @@
+--- !ruby/object:Gem::Specification
+name: gort
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Alexander Mankuta
+autorequire:
+bindir: bin
+cert_chain:
+- |
+  -----BEGIN CERTIFICATE-----
+  MIIC+jCCAeKgAwIBAgIBAzANBgkqhkiG9w0BAQsFADAjMSEwHwYDVQQDDBhhbGV4
+  L0RDPXBvaW50bGVzcy9EQz1vbmUwHhcNMjMxMTA5MTA1MzIxWhcNMjQxMTA4MTA1
+  MzIxWjAjMSEwHwYDVQQDDBhhbGV4L0RDPXBvaW50bGVzcy9EQz1vbmUwggEiMA0G
+  CSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDPOVLPGEK+eaP6zJfifrpWvPTg4qo3
+  XNJJPom80SwqX2hVCVsRDK4RYgKUQqKRQzHhlx14wZHwWLETBVbNDGX3uqyCnTWU
+  JUKh3ydiZShXpNHoV/NW7hhEYvNsDcBAjYTmbvXOhuYCo0Tz/0N2Oiun/0wIICtP
+  vytY9TY0/lklWjAbsqJjNOu3o8IYkJBAN/rU96E/6WhFwjnxLcTnV9RfFRXdjG5j
+  CughoB2xSwKX8gwbQ8fsnaZRmdyDGYNpz6sGF0zycfiLkTttbLA2nYATCALy98CH
+  nsyZNsTjb4WINCuY2yEDjwesw9f/ROkNC68EgQ5M+aMjp+D0WcYGfzojAgMBAAGj
+  OTA3MAkGA1UdEwQCMAAwCwYDVR0PBAQDAgSwMB0GA1UdDgQWBBRPgIwSVbeonua/
+  Ny/8576oxdUbrjANBgkqhkiG9w0BAQsFAAOCAQEAX28QLxNNz5EgaZZuQQUkbOXB
+  4b5luBO22535+Vgj2jw7yjV8KKoGMWKrnB00ijgntqPEPXCzaPNibOcPZV5WfWVS
+  t0Ls8lWE/8kezPwV4SbRe4Y7C+D4J+oirs0L5PtpREV9CJ7kfdW/AN9MtvjjBFlb
+  jHquD/MiOOMyHtuO0FiTL265m10thcAUsbyi0MehKgGbtJ5fGceHvZDqDouvbMjT
+  hoijFk1oTY939JhjdcHuJzMiS2TrqIw8Dr5DkQu2vAjHpw0aOOWhlRjNJ7RHYJNm
+  QugXmCnHQxSKTmc7imKuotyMdRRKFh8UEFCLRsFtBbNxkXyNuB4xBMuUYodhEw==
+  -----END CERTIFICATE-----
+date: 2024-06-22 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: addressable
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.8'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.8'
+- !ruby/object:Gem::Dependency
+  name: rchardet
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.8'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.8'
+description: robots.txt parser and evaluator according to RFC 9309.
+email:
+- alex@pointless.one
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- LICENSE.txt
+- lib/gort.rb
+- lib/gort/allow_rule.rb
+- lib/gort/disallow_rule.rb
+- lib/gort/group.rb
+- lib/gort/invalid_line.rb
+- lib/gort/parser.rb
+- lib/gort/path_rule.rb
+- lib/gort/robots_txt.rb
+- lib/gort/rule.rb
+- lib/gort/rule_set.rb
+- lib/gort/user_agent_rule.rb
+- lib/gort/version.rb
+homepage:
+licenses:
+- MIT
+metadata:
+  allowed_push_host: https://rubygems.org
+  rubygems_mfa_required: 'true'
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '3.1'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.5.9
+signing_key:
+specification_version: 4
+summary: robots.txt parser and evaluator.
+test_files: []

metadata.gz.sig

@@ -0,0 +1,2 @@
+f�*a��z0G���p��h:P3��H�I�wj�n�3
94��?z˩ޚ���H��p1��.������\0h&GP��� &I2ߪ��Ȕ˒49�Esp�f�>y�"$���p�J����Tќh�������=���2vZ������]G�.��T�ƹ�0<x�8su�V�nv�yI��L�B
+�F�������0ς��c��8�c�3��f6�e�JuA��Zg����%��í���LE:t�{��