probot 0.4.0 → 0.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 4f0fd52daf4e0752bc505a323bc25c87f2e80aa06c09a88652af1c0a1cb14281
-  data.tar.gz: 1f26a54f6a78deb956b0aaf120b8484ad926b1e7427f1a0bccb4e01b3a2e92cf
+  metadata.gz: 5a82d5e6f6baf600f7f6c22cf5358cefaed6ce9bf1119298c98245e440296c2e
+  data.tar.gz: c791777dfc59a614f7797b63765cb9d63d39ec732feab6df754025782e023825
 SHA512:
-  metadata.gz: 15076c129a6988ff8edb08dfdb5378de07daf9d81fb942bc4b6ffd4b0b752d3434ca357e5287e40d8242c0401fd1e85c72926d6b65f8df5f69dd0c5569bf5b9d
-  data.tar.gz: 462bd66a4030edac2b06c3ab95cdfe36e6bfc917e9ccd2ccbe8be8d9fd6c7c08867ce9bff20fc1e9daa123498159bf921023f0b020a36fd470274e129b63dfe5
+  metadata.gz: ada3169c4a078fe19a526ee07563ba355a6032d8674d70487db3ab6c14aabf65d518d989f3e3d5f6e7d4fbc55b1920b27d7b7828129412ae4d81f76b8f90df67
+  data.tar.gz: 9487fc153c81dc99ba73ec8a1330688f46b829e2e62db115e0a04651c1c41ca4418afe8cd22abc272272ef0e7dec70d1a81c0bcb4a36c0c7d3666eb7f32e43de

data/lib/probot/version.rb

@@ -1,3 +1,3 @@
 class Probot
-  VERSION = "0.4.0"
+  VERSION = "0.5.0"
 end

data/lib/probot.rb

@@ -31,6 +31,7 @@ class Probot
     @current_agents = ["*"]
     @current_agents.each { |agent| @rules[agent] ||= {"disallow" => [], "allow" => [], "crawl_delay" => 0} }
     @sitemaps = []
+
     @site = URI(data) if data.start_with?("http")
     @doc = @site.nil? ? data : fetch_robots_txt(@site)
     parse(@doc)
@@ -92,7 +93,9 @@ class Probot
 
       # All Regex characters are escaped, then we unescape * and $ as they may used in robots.txt
       if data.allow? || data.disallow?
-        @current_agents.each { |agent| rules[agent][data.key] << Regexp.new(Regexp.escape(data.value).gsub('\*', ".*").gsub('\$', "$")) }
+        @current_agents.each do |agent|
+          rules[agent][data.key] << Regexp.new(Regexp.escape(data.value).gsub('\*', ".*").gsub('\$', "$")) unless data.value.nil?
+        end
 
         # When user-agent strings are found on consecutive lines, they are considered to be part of the same record. Google ignores crawl_delay.
         subsequent_agent = false
@@ -128,6 +131,8 @@ class Probot
 
     def clean_value = raw_value.split("#").first&.strip
 
+    def clean_url = clean_value&.then { URI(_1).to_s }
+
     def agent? = key == "user-agent"
 
     def disallow? = key == "disallow"
@@ -140,11 +145,13 @@ class Probot
 
     def value
       return clean_value.to_f if crawl_delay?
-      return URI(clean_value).to_s if disallow? || allow?
+      return clean_url if disallow? || allow?
 
       raw_value
     rescue URI::InvalidURIError
       raw_value
+    rescue ArgumentError
+      raw_value
     end
   end
 

data/probot.gemspec

@@ -13,6 +13,7 @@ Gem::Specification.new do |spec|
   spec.homepage = "http://github.com/dkam/probot"
   spec.license = "MIT"
   spec.required_ruby_version = ">= 3.0"
+  spec.platform = Gem::Platform::RUBY
 
   spec.metadata["homepage_uri"] = spec.homepage
   spec.metadata["source_code_uri"] = "http://github.com/dkam/probot"
@@ -29,4 +30,5 @@ Gem::Specification.new do |spec|
   spec.bindir = "exe"
   spec.executables = spec.files.grep(%r{\Aexe/}) { |f| File.basename(f) }
   spec.require_paths = ["lib"]
+  spec.add_development_dependency "debug"
 end

metadata

@@ -1,15 +1,28 @@
 --- !ruby/object:Gem::Specification
 name: probot
 version: !ruby/object:Gem::Version
-  version: 0.4.0
+  version: 0.5.0
 platform: ruby
 authors:
 - Dan Milne
-autorequire:
 bindir: exe
 cert_chain: []
-date: 2024-10-30 00:00:00.000000000 Z
-dependencies: []
+date: 2024-12-23 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: debug
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 description: A fully featured robots.txt parser.
 email:
 - d@nmilne.com
@@ -33,7 +46,6 @@ metadata:
   homepage_uri: http://github.com/dkam/probot
   source_code_uri: http://github.com/dkam/probot
   changelog_uri: http://github.com/dkam/probot/CHANGELOG.md
-post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -48,8 +60,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.5.22
-signing_key:
+rubygems_version: 3.6.0
 specification_version: 4
 summary: A robots.txt parser.
 test_files: []