sitemap-parser 0.4.0 → 0.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: 5e1f45974fe39269287ae515e63e56d245dd0ff6
-  data.tar.gz: 99382aa0efafaccedf73189a741c6770ccf683a3
+SHA256:
+  metadata.gz: 8eddc8204f36a00114c47cbee1a9e1bf884fda408e51383e3af615c7811b1a25
+  data.tar.gz: 10fde984cc652d03edf863ee8c6cc03a531b6cadaa1e37de49b2d2095d3d4f46
 SHA512:
-  metadata.gz: 275725d0338f36502a147469b28a154e71e03af94d64d5f4947ef36d2a40dddb8707bdc83a97cbf2f582c9b17083a8160b023271754d67dafb76dc6cb6a83518
-  data.tar.gz: 1b000477be977c027646d887e7b5dc3595c029c01cf4467fbac8e79e92ceeae947259902b00761d071fbe656305deaeb8c3289063286a265fac44e3c27d32350
+  metadata.gz: ec5cad59821bdc90a202fca1ac7b514c34be55c4addd3948ba3b9fe86d68170dbb773836d8eb094f1b7625ee4112ea9903693edabb395b2951fa638a98d3083b
+  data.tar.gz: d2ffb35d82995f2e031afb304350f88a726db081937e9b955b689f774bf3c7e5a4ae8da132c95d6ef2f6edad5326a95a079194c048f01ae4e91d7878d0c51066

data/lib/sitemap-parser.rb

@@ -1,28 +1,29 @@
+# frozen_string_literal: true
+
 require 'nokogiri'
 require 'typhoeus'
+require 'zlib'
+require_relative 'sitemap-parser/version'
 
 class SitemapParser
-
   def initialize(url, opts = {})
     @url = url
-    @options = {:followlocation => true, :recurse => false}.merge(opts)
+    @options = { followlocation: true, recurse: false, url_regex: nil }.merge(opts)
   end
 
   def raw_sitemap
     @raw_sitemap ||= begin
-      if @url =~ /\Ahttp/i
-        request_options = @options.dup.tap { |opts| opts.delete(:recurse) }
+      if /\Ahttp/i.match?(@url)
+        request_options = @options.dup.tap { |opts| opts.delete(:recurse); opts.delete(:url_regex) }
         request = Typhoeus::Request.new(@url, request_options)
         request.on_complete do |response|
-          if response.success?
-            return response.body
-          else
-            raise "HTTP request to #{@url} failed"
-          end
+          raise "HTTP request to #{@url} failed" unless response.success?
+
+          return inflate_body_if_needed(response)
         end
         request.run
-      elsif File.exist?(@url) && @url =~ /[\\\/]sitemap\.xml\Z/i
-        open(@url) { |f| f.read }
+      elsif File.exist?(@url) && @url =~ %r{[\\/]sitemap\.xml\Z}i
+        File.open(@url, &:read)
       end
     end
   end
@@ -33,24 +34,44 @@ class SitemapParser
 
   def urls
     if sitemap.at('urlset')
-      sitemap.at("urlset").search("url")
+      filter_sitemap_urls(sitemap.at('urlset').search('url'))
     elsif sitemap.at('sitemapindex')
       found_urls = []
       if @options[:recurse]
-        sitemap.at('sitemapindex').search('sitemap').each do |sitemap|
+        urls = sitemap.at('sitemapindex').search('sitemap')
+        filter_sitemap_urls(urls).each do |sitemap|
           child_sitemap_location = sitemap.at('loc').content
-          found_urls << self.class.new(child_sitemap_location, :recurse => false).urls
+          found_urls << self.class.new(child_sitemap_location, recurse: false).urls
         end
       end
-      return found_urls.flatten
+      found_urls.flatten
     else
       raise 'Malformed sitemap, no urlset'
     end
   end
 
   def to_a
-    urls.map { |url| url.at("loc").content }
+    urls.map { |url| url.at('loc').content }
   rescue NoMethodError
     raise 'Malformed sitemap, url without loc'
   end
+
+  private
+
+  def filter_sitemap_urls(urls)
+    return urls if @options[:url_regex].nil?
+
+    urls.select { |url| url.at('loc').content.strip =~ @options[:url_regex] }
+  end
+
+  def inflate_body_if_needed(response)
+    return response.body unless response.headers
+
+    case response.headers['Content-Type']
+    when %r{application/gzip}, %r{application/x-gzip}, %r{application/octet-stream}
+      Zlib.gunzip(response.body)
+    else
+      response.body
+    end
+  end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: sitemap-parser
 version: !ruby/object:Gem::Version
-  version: 0.4.0
+  version: 0.5.0
 platform: ruby
 authors:
 - Ben Balter
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2017-07-31 00:00:00.000000000 Z
+date: 2020-12-01 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: nokogiri
@@ -28,16 +28,36 @@ dependencies:
   name: typhoeus
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0.6'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '2.0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0.6'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+- !ruby/object:Gem::Dependency
+  name: minitest
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '4.7'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '4.7'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
@@ -53,47 +73,61 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '10.4'
 - !ruby/object:Gem::Dependency
-  name: shoulda
+  name: rubocop
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.5'
+        version: '0.80'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.5'
+        version: '0.80'
 - !ruby/object:Gem::Dependency
-  name: bundler
+  name: rubocop-minitest
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.6'
+        version: '0.1'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.6'
+        version: '0.1'
 - !ruby/object:Gem::Dependency
-  name: minitest
+  name: rubocop-performance
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '4.7'
+        version: '1.5'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '4.7'
+        version: '1.5'
+- !ruby/object:Gem::Dependency
+  name: shoulda
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.5'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.5'
 - !ruby/object:Gem::Dependency
   name: test-unit
   requirement: !ruby/object:Gem::Requirement
@@ -119,7 +153,7 @@ homepage: https://github.com/benbalter/sitemap-parser
 licenses:
 - MIT
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -134,9 +168,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.6.11
-signing_key: 
+rubygems_version: 3.1.4
+signing_key:
 specification_version: 4
 summary: Ruby Gem to parse sitemaps.org compliant sitemaps
 test_files: []