probot 0.1.0 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 65dd2e16a696a2500937d32ae744262f74be414620e5505838fde49b8847d34d
-  data.tar.gz: 5f50733228c4c4eec37218eda00ceb4c2dd7eca801df6dd7345dd9e7edc94516
+  metadata.gz: 54e371737751fd5162dbcb07fbb7f24906f3d9684c17cec9c54f48ab5c99c35f
+  data.tar.gz: 5babb13f06601f07e58613649ddbc1396236c685e3858c4816f45fba95e95038
 SHA512:
-  metadata.gz: 33a44b9aba61643781e697e1b0fd54ac8d1afb40a61b1e2dc13d174aeea1b4ec1e6c2b762d122a2e1f5e50b447881b28c57cedaaff9a05f1174ce0d531c7f605
-  data.tar.gz: 85615bf3573de8af826308f5bc36df231581de0da4346081a44632cd5882df8c7574ce9889f586dc7fdde2c881a6da13edf49b806bc9f76d0676b20cd516a789
+  metadata.gz: c1250d28b604d70abedc9465c755f04e20c0318169fef2beab5c220692abadad073467130fd8f9d3c81ca443e3d659dded8eeec6af12403b594d9f3ad9a610af
+  data.tar.gz: 53f3d3fadba21111fd84c9a90501d22ea3456002966ce77ecff9757bff1d1544fce07d0971b3a30f0bf3683be21162771313ee6d9c1ed1b676fe11f63ccdbf03

data/CHANGELOG.md

@@ -1,5 +1,12 @@
 ## [Unreleased]
 
+## [0.3.0] - 2023-09-18
+
+- Only return unique sitemaps.
+
+## [0.2.0] - 2023-09-10
+
+- Correctly handle multiple sitemaps + tests.
 ## [0.1.0] - 2023-09-09
 
 - Initial release

data/lib/probot/version.rb

@@ -1,3 +1,3 @@
 class Probot
-  VERSION = "0.1.0"
+  VERSION = "0.3.0"
 end

data/lib/probot.rb

@@ -19,8 +19,8 @@ require "net/http"
 #   Find the most specific rule for a given URL. We use the length of the regexp as a proxy for specificity.
 
 class Probot
-  attr_reader :rules, :sitemap, :doc
-  attr_accessor :agent
+  attr_reader :rules, :doc
+  attr_accessor :agent, :sitemaps, :site
 
   def initialize(data, agent: "*")
     raise ArgumentError, "The first argument must be a string" unless data.is_a?(String)
@@ -30,8 +30,8 @@ class Probot
     @current_agents = ["*"]
     @current_agents.each { |agent| @rules[agent] ||= {"disallow" => [], "allow" => [], "crawl_delay" => 0} }
     @sitemaps = []
-
-    @doc = data.start_with?("http") ? fetch_robots_txt(data) : data
+    @site = URI(data) if data.start_with?("http")
+    @doc = @site.nil? ? data : fetch_robots_txt(@site)
     parse(@doc)
   end
 
@@ -90,11 +90,11 @@ class Probot
       end
 
       # All Regex characters are escaped, then we unescape * and $ as they may used in robots.txt
-
       if data.allow? || data.disallow?
         @current_agents.each { |agent| rules[agent][data.key] << Regexp.new(Regexp.escape(data.value).gsub('\*', ".*").gsub('\$', "$")) }
 
-        subsequent_agent = false # When user-agent strings are found on consecutive lines, they are considered to be part of the same record. Google ignores crawl_delay.
+        # When user-agent strings are found on consecutive lines, they are considered to be part of the same record. Google ignores crawl_delay.
+        subsequent_agent = false
         next
       end
 
@@ -103,8 +103,12 @@ class Probot
         next
       end
 
+      # Ensure we have an absolute URL
       if data.sitemap?
-        @sitemap = URI(data.value).path
+        sitemap_uri = URI(data.value)
+        sitemap_uri = sitemap_uri.host.nil? ? URI.join(*[site, sitemap_uri].compact) : sitemap_uri
+        @sitemaps << sitemap_uri.to_s
+        @sitemaps.uniq!
         next
       end
 

data/probot.gemspec

@@ -29,10 +29,4 @@ Gem::Specification.new do |spec|
   spec.bindir = "exe"
   spec.executables = spec.files.grep(%r{\Aexe/}) { |f| File.basename(f) }
   spec.require_paths = ["lib"]
-
-  # Uncomment to register a new dependency of your gem
-  # spec.add_dependency "example-gem", "~> 1.0"
-
-  # For more information and examples about making a new gem, check out our
-  # guide at: https://bundler.io/guides/creating_gem.html
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: probot
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.3.0
 platform: ruby
 authors:
 - Dan Milne
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2023-09-10 00:00:00.000000000 Z
+date: 2023-09-18 00:00:00.000000000 Z
 dependencies: []
 description: A fully featured robots.txt parser.
 email: