sitemaps_parser 0.2.0 → 0.2.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: f6a5cf10a4446ce42e918767ee0279a81178951c
-  data.tar.gz: 878301d4d7ffd4429a0d4763d9ecb4ccca719039
+  metadata.gz: 48e74e772cd689a18f735341f9cc1272494e26b8
+  data.tar.gz: 12737b00e5a24ae20ccb17c079b3fc3621e555c8
 SHA512:
-  metadata.gz: 1a209890b6494fb17fed6c234656854a657dec8dc5d6012d752d6962339b8b5c06f8cd0facb15c96884f18bcabf4bb2a56aea53b84ad8326695ae43c4e57fe2b
-  data.tar.gz: 108c423e4969c262bf00959fa6ae4d5855c0d35faf10d21c282d3a05413e2f4e9178767dfce86da7872231dc48ac7a81338c72a6b1f809d73ff2d7e3e11a7c9d
+  metadata.gz: 98d19dfb44b860c00401bcf2318e6a077101cd27a11fa7665d8d4b2d1a5bcbdfe11d0bd43e9b1da1e5e378dbe9eb92898b8132f2e6fe883e4df5fc0a0ae94138
+  data.tar.gz: e966d408e8edb14f66fce1ad6ce4f3053a53667a9e56ef26d3d3375311b3b4b83a794da859b174c67532dddaa4d01e6fb22cb0d5ec97bd0c856f2190bd0e4fb9

data/lib/sitemaps/parser.rb

@@ -9,9 +9,10 @@ module Sitemaps
     # @param source [String] an XML string to parse.
     # @param max_entries [Integer, nil] the maximum number of entries to add to the sitemap.
     # @param filter [#call, nil] if provided, called per entry to filter the entry out of the sitemap.
+    # @param filter_indexes [Boolean, nil] if truthy, filter is called per submap as well as entries.
     # @return [Sitemap] the sitemap parsed from the XML string. If the XML string given is invalid,
     #   a sitemap will still be returned, but the entries and sitemaps keys will be empty.
-    def self.parse(source, max_entries: nil, filter: nil)
+    def self.parse(source, max_entries: nil, filter: nil, filter_indexes: nil)
       document = REXML::Document.new(source)
       entries  = document.elements.to_a("/urlset/url").map do |root|
         loc  = parse_loc(root) || next
@@ -29,7 +30,8 @@ module Sitemaps
         loc  = parse_loc(root) || next
         mod  = parse_lastmod(root)
 
-        Sitemaps::Submap.new(loc, mod)
+        submap = Sitemaps::Submap.new(loc, mod)
+        (!filter || !filter_indexes || filter.call(submap)) ? submap : nil
       end.reject(&:nil?)
 
       Sitemaps::Sitemap.new(entries, sitemaps)

data/lib/sitemaps/version.rb

@@ -1,3 +1,3 @@
 module Sitemaps
-  VERSION = "0.2.0".freeze
+  VERSION = "0.2.1".freeze
 end

data/lib/sitemaps.rb

@@ -50,7 +50,7 @@ module Sitemaps
   #     a good idea to include, as many sites have _very_ large sitemaps.
   #   @return [Sitemap]
   #
-  # @overload fetch(url, fetcher: nil, max_entries: nil)
+  # @overload fetch(url, fetcher: nil, filter_indexes: nil, max_entries: nil)
   #   If a block is given, it's used as a filter for entries before they're added to the sitemap.
   #
   #   @param url [String, URI] the url of the sitemap in question.
@@ -58,17 +58,19 @@ module Sitemaps
   #   @param max_entries [Integer] the maximum number of entries to include in the sitemap. Once the
   #     sitemap has this many entries, further fetches and parsing will not occur. This is always
   #     a good idea to include, as many sites have _very_ large sitemaps.
+  #   @param filter_indexes [Boolean] if true, Submap instances will be run through the filter block
+  #     as well as Entry instances.
   #   @return [Sitemap]
   #   @yield [Entry] Filters the entry from the sitemap if the block returns falsey.
   #   @yieldreturn [Boolean] whether or not to include the entry in the sitemap.
-  def self.fetch(url, fetcher: nil, max_entries: nil, &block)
+  def self.fetch(url, fetcher: nil, max_entries: nil, filter_indexes: nil, &block)
     fetcher ||= @default_fetcher
     unless url.is_a? URI
       url = "http://#{url}" unless url =~ %r{^https?://}
       url = URI.parse(url)
     end
 
-    _instance.fetch_recursive(url, fetcher, max_entries, &block)
+    _instance.fetch_recursive(url, fetcher, max_entries, filter_indexes, &block)
   end
 
   # Discover, fetch and parse sitemaps from the given host.
@@ -93,10 +95,12 @@ module Sitemaps
   #   @param max_entries [Integer] the maximum number of entries to include in the sitemap. Once the
   #     sitemap has this many entries, further fetches and parsing will not occur. This is always
   #     a good idea to include, as many sites have _very_ large sitemaps.
+  #   @param filter_indexes [Boolean] if true, Submap instances will be run through the filter block
+  #     as well as Entry instances.
   #   @return [Sitemap]
   #   @yield [Entry] Filters the entry from the sitemap if the block returns falsey.
   #   @yieldreturn [Boolean] whether or not to include the entry in the sitemap.
-  def self.discover(url, fetcher: nil, max_entries: nil, &block)
+  def self.discover(url, fetcher: nil, max_entries: nil, filter_indexes: nil, &block)
     fetcher ||= @default_fetcher
     unless url.is_a? URI
       url = "http://#{url}" unless url =~ %r{^https?://}
@@ -104,7 +108,7 @@ module Sitemaps
     end
 
     roots = _instance.discover_roots(url, fetcher)
-    _instance.fetch_recursive(roots, fetcher, max_entries, &block)
+    _instance.fetch_recursive(roots, fetcher, max_entries, filter_indexes, &block)
   end
 
   # @return [Instance]
@@ -120,7 +124,7 @@ module Sitemaps
   class Instance
     # recursively fetch sitemaps and sitemap indexes from the given urls.
     # @return [Sitemap]
-    def fetch_recursive(urls, fetcher, max_entries, &block)
+    def fetch_recursive(urls, fetcher, max_entries, filter_indexes, &block)
       queue = urls.is_a?(Array) ? urls : [urls]
       maps  = {}
 
@@ -134,7 +138,7 @@ module Sitemaps
 
           # fetch this item in the queue, and queue up any sub maps it found
           source  = fetcher.call(url)
-          sitemap = Sitemaps::Parser.parse(source, max_entries: max_entries, filter: block)
+          sitemap = Sitemaps::Parser.parse(source, max_entries: max_entries, filter: block, filter_indexes: filter_indexes)
 
           # save the results and queue up any submaps it found
           maps[url] = sitemap

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: sitemaps_parser
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.2.1
 platform: ruby
 authors:
 - Jonathan Raphaelson
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-03-24 00:00:00.000000000 Z
+date: 2016-03-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler