sitemap-parser 0.2.1 → 0.5.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: d03dd8db51f6b4d2f7fc07f1bf82a8208e09879e
-  data.tar.gz: 36e811e0184838fdb0332fc5914b5ef232f6ee87
+SHA256:
+  metadata.gz: b98303437a057f9f3cdbe7ef5ef3712ed6534e8d9e228364a89bac75488e27df
+  data.tar.gz: 380dfe60cde2614823adaaa87520306b5689424d882f6134d80920b6947f717e
 SHA512:
-  metadata.gz: 49c136b2f86b00dffbf9c565a7284ea1349a68f752dae6e1cab96b8135ab69955091f9fbb2fbd2d32880d3bc5cde7b8a29bf8eca42c2e954f09a4f4a306d7e76
-  data.tar.gz: d5241537a66a454a6097a6c88bc3e1e643535c0a085d98f935c13aea902c5b5bf094042fbc2a140c437d548e1e9e9864dda05f21a42113452ce811499b7bd7cc
+  metadata.gz: d6afba1d0068955ccf40bb147ddb97349dd8b04499dbe755b04e03b89b8188ab0a37bf618ca96ad49908f4a850fd230d3c00ff24066881044d12e222ef415221
+  data.tar.gz: 43e3baffe31bb3dc672d2c73c8002a2707721551be1efd1314a03818038b4a5afb3350ad8c507dfc884c25501982c84297f4c20a207019dc965e9b27a3b82941

data/lib/sitemap-parser/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+class SitemapParser
+  VERSION = '0.5.6'
+end

data/lib/sitemap-parser.rb

@@ -1,46 +1,77 @@
+# frozen_string_literal: true
+
 require 'nokogiri'
 require 'typhoeus'
+require 'zlib'
+require_relative 'sitemap-parser/version'
 
 class SitemapParser
-
   def initialize(url, opts = {})
     @url = url
-    @options = {:followlocation => true}.merge(opts)
+    @options = { followlocation: true, recurse: false, url_regex: nil }.merge(opts)
   end
 
   def raw_sitemap
     @raw_sitemap ||= begin
-      if @url =~ /\Ahttp/i
-        request = Typhoeus::Request.new(@url, followlocation: @options[:followlocation])
+      if /\Ahttp/i.match?(@url)
+        request_options = @options.dup.tap { |opts| opts.delete(:recurse); opts.delete(:url_regex) }
+        request = Typhoeus::Request.new(@url, request_options)
         request.on_complete do |response|
-          if response.success?
-            return response.body
-          else
-            return nil
-          end
+          raise "HTTP request to #{@url} failed" unless response.success?
+
+          return inflate_body_if_needed(response)
         end
         request.run
-      elsif File.exist?(@url) && @url =~ /[\\\/]sitemap\.xml\Z/i
-        open(@url) { |f| f.read }
+      elsif File.exist?(@url) && @url =~ %r{[\\/]sitemap\.xml\Z}i
+        File.open(@url, &:read)
       end
     end
   end
 
   def sitemap
     @sitemap ||= Nokogiri::XML(raw_sitemap)
-  rescue
-    nil
   end
 
   def urls
-    sitemap.at("urlset").search("url")
-  rescue
-    nil
+    if sitemap.at('urlset')
+      filter_sitemap_urls(sitemap.at('urlset').search('url'))
+    elsif sitemap.at('sitemapindex')
+      found_urls = []
+      if @options[:recurse]
+        urls = sitemap.at('sitemapindex').search('sitemap')
+        filter_sitemap_urls(urls).each do |sitemap|
+          child_sitemap_location = sitemap.at('loc').content
+          found_urls << self.class.new(child_sitemap_location, recurse: false).urls
+        end
+      end
+      found_urls.flatten
+    else
+      raise 'Malformed sitemap, no urlset'
+    end
   end
 
   def to_a
-    urls.map { |url| url.at("loc").content }
-  rescue
-    []
+    urls.map { |url| url.at('loc').content }
+  rescue NoMethodError
+    raise 'Malformed sitemap, url without loc'
+  end
+
+  private
+
+  def filter_sitemap_urls(urls)
+    return urls if @options[:url_regex].nil?
+
+    urls.select { |url| url.at('loc').content.strip =~ @options[:url_regex] }
+  end
+
+  def inflate_body_if_needed(response)
+    return response.body unless response.headers
+
+    case response.headers['Content-Type']
+    when %r{application/gzip}, %r{application/x-gzip}, %r{application/octet-stream}
+      Zlib.gunzip(response.body)
+    else
+      response.body
+    end
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: sitemap-parser
 version: !ruby/object:Gem::Version
-  version: 0.2.1
+  version: 0.5.6
 platform: ruby
 authors:
 - Ben Balter
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2015-01-05 00:00:00.000000000 Z
+date: 2021-12-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: nokogiri
@@ -28,16 +28,36 @@ dependencies:
   name: typhoeus
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0.6'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '2.0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0.6'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+- !ruby/object:Gem::Dependency
+  name: minitest
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '4.7'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '4.7'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
@@ -53,47 +73,75 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '10.4'
 - !ruby/object:Gem::Dependency
-  name: shoulda
+  name: rubocop
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.5'
+        version: '0.80'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.5'
+        version: '0.80'
 - !ruby/object:Gem::Dependency
-  name: bundler
+  name: rubocop-minitest
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.6'
+        version: '0.1'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.6'
+        version: '0.1'
 - !ruby/object:Gem::Dependency
-  name: minitest
+  name: rubocop-performance
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '4.7'
+        version: '1.5'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '4.7'
+        version: '1.5'
+- !ruby/object:Gem::Dependency
+  name: shoulda
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.5'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.5'
+- !ruby/object:Gem::Dependency
+  name: test-unit
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.1'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.1'
 description: Ruby Gem to parse sitemaps.org compliant sitemaps.
 email: ben.balter@github.com
 executables: []
@@ -101,11 +149,12 @@ extensions: []
 extra_rdoc_files: []
 files:
 - lib/sitemap-parser.rb
+- lib/sitemap-parser/version.rb
 homepage: https://github.com/benbalter/sitemap-parser
 licenses:
 - MIT
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -120,9 +169,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.2.0
-signing_key: 
+rubygems_version: 3.2.22
+signing_key:
 specification_version: 4
 summary: Ruby Gem to parse sitemaps.org compliant sitemaps
 test_files: []