sitemaps_parser 0.1.0 → 0.1.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 84077b93f96854c50072c350407fb253b9e0a952
-  data.tar.gz: 6b1437162a1e98e3e882815cabea1ab5e4b1dc90
+  metadata.gz: 2566aba4212963304e040f2d4867b40b0d263707
+  data.tar.gz: 8f9bc5c86db26122c47e68b83ec14ff34bb76e5a
 SHA512:
-  metadata.gz: 1e705e71f275f333a98dee425c0c11eddcab08b502233e0500328e926f3570227549ef7867206ad03dcd292a932b477b71592123b7ab456594c981dac5e624fb
-  data.tar.gz: 0decafa6d29c0785d01f2099c64691678df91123986e5f075decf3c5b09d55738195aa6d53c787dfbd757b39462c4df245711c5b1ce90640a88afdc89a28854c
+  metadata.gz: aabccdfa4ac6719e0347cc29f100125b0e1fbd625021853b2cd81b54aa7a6daff29ab84761d5826db5ff14fb412fa1fa6a12befe4daeb8bf142a06c3ab329100
+  data.tar.gz: ed64e0d67a466a6f4081d7cbe5e3f00b558188ceb9bde1139fa8917859f5ac0d3f1675403829ae309149d3ea988a2825ebeef7e66572106ca10c552171074c17

data/README.md

@@ -1,7 +1,15 @@
 # Sitemaps
 
+[![Gem](https://img.shields.io/gem/v/sitemaps_parser.svg?style=flat-square)](https://rubygems.org/gems/sitemaps_parser)
+
 Discover, retrieve and parse XML sitemaps, according to the spec at [sitemaps.org](http://sitemaps.org).
 
+## TODO
+
+* discovery
+* sorting by last modified, or priority
+* filtering by last modified
+
 ## Installation
 
 Add this line to your application's Gemfile:
@@ -27,19 +35,15 @@ require 'sitemaps'
 Sitemaps.parse("<xml ns=\"...")
 
 # fetch and parse a sitemap from a known url
-Sitemaps.fetch("http://google.com/sitemap.xml", recurse: true, fetch: :builtin)
+sitemap = Sitemaps.fetch("http://google.com/sitemap.xml", recurse: true)
 
-# attempt to discover a sitemap url for a domain
-# parallel fetches a small set of 'normal' urls, returns whichever comes back first, most likely only one.
-xml = Sitemaps.discover("http://example.com")
-if url.present?
-  sitemap = Sitemaps.parse(xml)
+# fetch and parse sitemaps, excluding paths matching a filter, and limiting to the top 200 
+sitemap = Sitemaps.fetch("https://www.digitalocean.com/sitemaps.xml.gz", max_entries: 200) do |entry|
+  entry.loc.path !~ /blog/i
 end
 
 # sitemap usage
-
 sitemap.entries.first #> Struct(loc: 'http://example.com/page', lastmod: DateTime.utc, changefreq: :monthly, priority: 0.5)
-
 urls = sitemap.entries.map(&:loc)
 ```
 

data/lib/sitemaps/fetcher.rb

@@ -20,7 +20,7 @@ module Sitemaps
             return resp.body
           end
 
-          # on a 3xx response, handle the redirect
+        # on a 3xx response, handle the redirect
         elsif resp.code.to_s =~ /3\d\d/
           location = URI.parse(resp.header['location'])
           location = uri + resp.header['location'] if location.relative?
@@ -29,14 +29,14 @@ module Sitemaps
           attempts += 1
           next
 
-          # otherwise (4xx, 5xx) throw an exception
+        # otherwise (4xx, 5xx) throw an exception
         else
-          fail FetchError, "Failed to fetch URI, #{uri}, failed with response code: #{resp.code}"
+          raise FetchError, "Failed to fetch URI, #{uri}, failed with response code: #{resp.code}"
         end
       end
 
       # if we got here, we ran out of attempts
-      fail MaxRedirectError, "Failed to fetch URI #{uri}, redirected too many times" if attempts >= @max_attempts
+      raise MaxRedirectError, "Failed to fetch URI #{uri}, redirected too many times" if attempts >= @max_attempts
     end
   end
 end

data/lib/sitemaps/parser.rb

@@ -1,12 +1,7 @@
 module Sitemaps
   # Parse XML Sitemaps
   module Parser
-    require "time"
-    require "rexml/document"
-    require "active_support"
-    require "active_support/core_ext/object/try"
-
-    def self.parse(source)
+    def self.parse(source, max_entries: nil, filter: nil)
       document = REXML::Document.new(source)
       entries  = document.elements.to_a("/urlset/url").map do |root|
         loc  = parse_loc(root) || next
@@ -14,8 +9,10 @@ module Sitemaps
         freq = parse_changefreq(root)
         pri  = parse_priority(root)
 
-        Sitemaps::Entry.new(loc, mod, freq, pri)
+        entry = Sitemaps::Entry.new(loc, mod, freq, pri)
+        (!filter || filter.call(entry)) ? entry : nil
       end.reject(&:nil?)
+      entries = entries.take(max_entries) unless max_entries.nil?
 
       sitemaps = document.elements.to_a("/sitemapindex/sitemap").map do |root|
         loc  = parse_loc(root) || next

data/lib/sitemaps/version.rb

@@ -1,3 +1,3 @@
 module Sitemaps
-  VERSION = "0.1.0".freeze
+  VERSION = "0.1.1".freeze
 end

data/lib/sitemaps.rb

@@ -1,5 +1,14 @@
+require "active_support"
+require "active_support/core_ext/object/try"
 require "active_support/core_ext/object/blank"
 
+require "time"
+require "rexml/document"
+
+require "sitemaps/version"
+require "sitemaps/parser"
+require "sitemaps/fetcher"
+
 # Discover, fetch and parse XML sitemaps as defined by the `http://sitemaps.org` spec.
 module Sitemaps
   Entry   = Struct.new(:loc, :lastmod, :changefreq, :priority)
@@ -10,18 +19,22 @@ module Sitemaps
     Sitemaps::Parser.parse(source)
   end
 
-  def self.fetch(url, fetch: nil, recurse: true)
-    fetch ||= -> (url) { Sitemaps::Fetcher.fetch(url) }
-    recurse ? fetch_recursive(url, fetch) : fetch_single(url, fetch)
+  def self.fetch(url, fetch: nil, recurse: true, max_entries: nil, &block)
+    fetch ||= -> (u) { Sitemaps::Fetcher.fetch(u) }
+    url     = parse_url(url)
+
+    recurse ? fetch_recursive(url, fetch, max_entries, &block) : fetch_single(url, fetch, max_entries, &block)
   end
 
-  def self.fetch_single(url, fetch)
-    source = fetch.call(parse_url(url))
-    Sitemaps::Parser.parse(source)
+  def self.fetch_single(url, fetch, max_entries, &block)
+    url    = parse_url(url)
+    source = fetch.call(url)
+
+    Sitemaps::Parser.parse(source, max_entries: max_entries, filter: block)
   end
 
-  def self.fetch_recursive(url, fetch)
-    queue = [url]
+  def self.fetch_recursive(url, fetch, max_entries, &block)
+    queue = [parse_url(url)]
     maps  = {}
 
     # walk the queue, fetching the sitemap requested and adding
@@ -32,8 +45,15 @@ module Sitemaps
         break if url.nil?
         next  unless maps[url].nil?
 
-        maps[url] = fetch_single(url, fetch)
+        # fetch this item in the queue, and queue up any sub maps it found
+        maps[url] = fetch_single(url, fetch, max_entries, &block)
         queue.push(*maps[url].sitemaps.map(&:loc))
+
+        # decrement max_entries (since it's max_entries total, not per map)
+        unless max_entries.nil?
+          max_entries -= maps[url].entries.length
+          break if max_entries <= 0
+        end
       rescue => ex
         $stderr.puts "ERROR FETCHING: #{url}, #{ex.message}, ignoring..."
         next
@@ -50,11 +70,7 @@ module Sitemaps
   def self.parse_url(url)
     return url if url.is_a? URI
 
-    url = "http://#{url}" unless url =~ /^https?/
+    url = "http://#{url}" unless url =~ %r{^https?://}
     URI.parse(url)
   end
 end
-
-require "sitemaps/version"
-require "sitemaps/parser"
-require "sitemaps/fetcher"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: sitemaps_parser
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.1.1
 platform: ruby
 authors:
 - Jonathan Raphaelson
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-03-21 00:00:00.000000000 Z
+date: 2016-03-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler