sitemap_checker 0.0.1 → 0.0.2

data/README.md

@@ -1,6 +1,6 @@
 # SitemapChecker
 
-Checks sitemap urls for valid response
+Takes a url pointing to an xml or xml.gz sitemap or siteindex file and returns array of status messages for urls contained within.
 
 ## Installation
 
@@ -18,7 +18,8 @@ Or install it yourself as:
 
 ## Usage
 
-TODO: Write usage instructions here
+    SitemapChecker::Checker.new(url)
+
 
 ## Contributing
 

data/lib/sitemap_checker.rb

@@ -7,35 +7,44 @@ module SitemapChecker
   class Checker
     attr_reader :status_list
 
-    def initialize(url,schema='http://www.sitemaps.org/schemas/sitemap/0.9/sitemap.xsd')
-      @schema = schema
+    def initialize(url,schema='')
       @url = url
-      @sitemap = get_xml_from_url
-      sitemap_is_valid?
-      @status_list = get_status_list
+      @status_list = Array.new
+      process_xml
     end
 
     private
 
-    def get_xml_from_url
+    def get_xml_from_url(url)
       begin
-        Nokogiri::XML(Zlib::GzipReader.new(open(@url)))
+        Nokogiri::XML(Zlib::GzipReader.new(open(url)))
       rescue
-        Nokogiri::XML(open(@url))
+        Nokogiri::XML(open(url))
       end
     end
 
-    def sitemap_is_valid?
-      xsd = Nokogiri::XML::Schema(open(@schema))
-      raise 'Invalid Schema' unless xsd.valid?(@sitemap)
-      true
+    def process_xml
+      mxsd = Nokogiri::XML::Schema(open('http://www.sitemaps.org/schemas/sitemap/0.9/sitemap.xsd'))
+      ixsd = Nokogiri::XML::Schema(open('http://www.sitemaps.org/schemas/sitemap/0.9/siteindex.xsd'))
+      xml = get_xml_from_url(@url)
+      if mxsd.valid?(xml)
+        @status_list = get_status_list(urls(xml))
+      elsif ixsd.valid?(xml)
+        maps = urls(xml)
+        maps.each do |map|
+          xml = get_xml_from_url(map)
+          @status_list += get_status_list(urls(xml))
+        end
+      else raise 'Invalid Schema'
+        false
+      end
     end
 
-    def urls
-      @sitemap.xpath("//xmlns:loc")
+    def urls(xml)
+      xml.xpath("//xmlns:loc")
     end
 
-    def get_status_list
+    def get_status_list(urls)
       statuses = []
       urls.each do |url|
         begin

data/lib/sitemap_checker/version.rb

@@ -1,3 +1,3 @@
 module SitemapChecker
-  VERSION = "0.0.1"
+  VERSION = "0.0.2"
 end

data/spec/fixtures/siteindex.xml

@@ -0,0 +1,11 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<sitemapindex xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://www.sitemaps.org/schemas/sitemap/0.9 http://www.sitemaps.org/schemas/sitemap/0.9/siteindex.xsd" xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
+  <sitemap>
+    <loc>http://www.github.com/sitemap.xml</loc>
+    <lastmod>2012-07-30T17:14:40-07:00</lastmod>
+  </sitemap>
+  <sitemap>
+    <loc>http://www.github.com/sitemap.xml.gz</loc>
+    <lastmod>2012-07-30T17:14:40-07:00</lastmod>
+  </sitemap>
+</sitemapindex>

data/spec/fixtures/siteindex.xsd

@@ -0,0 +1,73 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<xsd:schema xmlns:xsd="http://www.w3.org/2001/XMLSchema"
+            targetNamespace="http://www.sitemaps.org/schemas/sitemap/0.9"
+            xmlns="http://www.sitemaps.org/schemas/sitemap/0.9"
+            elementFormDefault="qualified">
+<xsd:annotation>
+  <xsd:documentation>
+    XML Schema for Sitemap index files.
+    Last Modifed 2009-04-08
+  </xsd:documentation>
+</xsd:annotation>
+
+<xsd:element name="sitemapindex">
+  <xsd:annotation>
+    <xsd:documentation>
+      Container for a set of up to 50,000 sitemap URLs.
+      This is the root element of the XML file.
+    </xsd:documentation>
+  </xsd:annotation>
+  <xsd:complexType>
+    <xsd:sequence>
+      <xsd:element name="sitemap" type="tSitemap" maxOccurs="unbounded"/>
+    </xsd:sequence>
+  </xsd:complexType>
+</xsd:element>
+
+<xsd:complexType name="tSitemap">
+  <xsd:annotation>
+    <xsd:documentation>
+      Container for the data needed to describe a sitemap.
+    </xsd:documentation>
+  </xsd:annotation>
+  <xsd:all>
+    <xsd:element name="loc" type="tLocSitemap"/>
+    <xsd:element name="lastmod" type="tLastmodSitemap" minOccurs="0"/>
+  </xsd:all>
+</xsd:complexType>
+
+<xsd:simpleType name="tLocSitemap">
+  <xsd:annotation>
+    <xsd:documentation>
+      REQUIRED: The location URI of a sitemap.
+      The URI must conform to RFC 2396 (http://www.ietf.org/rfc/rfc2396.txt).
+    </xsd:documentation>
+  </xsd:annotation>
+  <xsd:restriction base="xsd:anyURI">
+    <xsd:minLength value="12"/>
+    <xsd:maxLength value="2048"/>
+  </xsd:restriction>
+</xsd:simpleType>
+
+<xsd:simpleType name="tLastmodSitemap">
+  <xsd:annotation>
+    <xsd:documentation>
+      OPTIONAL: The date the document was last modified. The date must conform
+      to the W3C DATETIME format (http://www.w3.org/TR/NOTE-datetime).
+      Example: 2005-05-10
+      Lastmod may also contain a timestamp.
+      Example: 2005-05-10T17:33:30+08:00
+    </xsd:documentation>
+  </xsd:annotation>
+  <xsd:union>
+    <xsd:simpleType>
+      <xsd:restriction base="xsd:date"/>
+    </xsd:simpleType>
+    <xsd:simpleType>
+      <xsd:restriction base="xsd:dateTime"/>
+      </xsd:simpleType>
+  </xsd:union>
+</xsd:simpleType>
+
+
+</xsd:schema>

data/spec/sitemap_checker_spec.rb

@@ -8,16 +8,24 @@ describe SitemapChecker do
     @dir  = Pathname.new(File.dirname(__FILE__))
     stub_request(:any, "http://www.github.com").to_return(:status => 200, :body => 'foo')
     stub_request(:any, "http://www.github.com/404").to_return(:status => 404, :body => 'foo')
-    stub_request(:any, "http://www.github.com/index.xml").to_return(:status => 200, :body => File.read(@dir + 'fixtures/valid_sitemap.xml'))
-    stub_request(:any, "http://www.github.com/index.xml.gz").to_return(:status => 200, :body => File.read(@dir + 'fixtures/valid_sitemap.xml.gz'))
-    stub_request(:get, "http://www.sitemaps.org/schemas/sitemap/0.9/sitemap.xsd").
-      with(:headers => {'Accept'=>'*/*', 'User-Agent'=>'Ruby'}).
-      to_return(:status => 200, :body => File.read(@dir + 'fixtures/sitemap_schema.xml'), :headers => {})
+    stub_request(:any, "http://www.github.com/sitemap.xml").to_return(:status => 200, :body => File.read(@dir + 'fixtures/sitemap.xml'))
+    stub_request(:any, "http://www.github.com/sitemap.xml.gz").to_return(:status => 200, :body => File.read(@dir + 'fixtures/sitemap.xml.gz'))
+    stub_request(:any, "http://www.github.com/siteindex.xml").to_return(:status => 200, :body => File.read(@dir + 'fixtures/siteindex.xml'))
+    stub_request(:any, "http://www.github.com/siteindex.xml.gz").to_return(:status => 200, :body => File.read(@dir + 'fixtures/siteindex.xml.gz'))
+    stub_request(:get, "http://www.sitemaps.org/schemas/sitemap/0.9/sitemap.xsd").to_return(:status => 200, :body => File.read(@dir + 'fixtures/sitemap.xsd'), :headers => {})
+    stub_request(:get, "http://www.sitemaps.org/schemas/sitemap/0.9/siteindex.xsd").to_return(:status => 200, :body => File.read(@dir + 'fixtures/siteindex.xsd'), :headers => {})
   end
 
-  it "accepts xml and gzipped files" do
-    @xml_sitemap = SitemapChecker::Checker.new('http://www.github.com/index.xml')
-    @gz_sitemap = SitemapChecker::Checker.new('http://www.github.com/index.xml.gz')
+  it "accepts xml and gzipped siteindexes" do
+    @xml_sitemap = SitemapChecker::Checker.new('http://www.github.com/siteindex.xml')
+    @gz_sitemap = SitemapChecker::Checker.new('http://www.github.com/siteindex.xml.gz')
+    @xml_sitemap.status_list.size.should eq(4)
+    @gz_sitemap.status_list.size.should eq(4)
+  end
+
+  it "accepts xml and gzipped sitemaps" do
+    @xml_sitemap = SitemapChecker::Checker.new('http://www.github.com/sitemap.xml')
+    @gz_sitemap = SitemapChecker::Checker.new('http://www.github.com/sitemap.xml.gz')
     @xml_sitemap.status_list.size.should eq(2)
     @gz_sitemap.status_list.size.should eq(2)
   end
@@ -26,9 +34,14 @@ describe SitemapChecker do
     lambda {SitemapChecker::Checker.new('http://www.github.com')}.should raise_error(RuntimeError, 'Invalid Schema')
   end
 
-  it "returns list of urls with responses" do
-    @valid_sitemap = SitemapChecker::Checker.new('http://www.github.com/index.xml')
-    @valid_sitemap.status_list.should eq([['http://www.github.com','200'], ['http://www.github.com/404','404']])
+  it "returns list of urls with responses from sitemap" do
+    @sitemap = SitemapChecker::Checker.new('http://www.github.com/sitemap.xml')
+    @sitemap.status_list.should eq([['http://www.github.com','200'], ['http://www.github.com/404','404']])
+  end
+
+  it "returns list of urls with responses from siteindex" do
+    @siteindex = SitemapChecker::Checker.new('http://www.github.com/siteindex.xml')
+    @siteindex.status_list.should eq([['http://www.github.com','200'], ['http://www.github.com/404','404'], ['http://www.github.com','200'], ['http://www.github.com/404','404']])
   end
 
 end

metadata

@@ -1,39 +1,46 @@
---- !ruby/object:Gem::Specification
+--- !ruby/object:Gem::Specification 
 name: sitemap_checker
-version: !ruby/object:Gem::Version
-  version: 0.0.1
+version: !ruby/object:Gem::Version 
+  hash: 1317335842608397244
   prerelease: 
+  segments: 
+  - 0
+  - 0
+  - 2
+  version: 0.0.2
 platform: ruby
-authors:
+authors: 
 - Gerlando Piro
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-07-29 00:00:00.000000000 Z
-dependencies:
-- !ruby/object:Gem::Dependency
+
+date: 2012-07-31 00:00:00 Z
+dependencies: 
+- !ruby/object:Gem::Dependency 
   name: nokogiri
-  requirement: !ruby/object:Gem::Requirement
-    none: false
-    requirements:
-    - - ! '>='
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :runtime
   prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
+  requirement: &id001 !ruby/object:Gem::Requirement 
     none: false
-    requirements:
-    - - ! '>='
-      - !ruby/object:Gem::Version
-        version: '0'
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 2002549777813010636
+        segments: 
+        - 0
+        version: "0"
+  type: :runtime
+  version_requirements: *id001
 description: SiteMap Checker
-email:
+email: 
 - gerlando@gmail.com
 executables: []
+
 extensions: []
+
 extra_rdoc_files: []
-files:
+
+files: 
 - .gitignore
 - Gemfile
 - LICENSE
@@ -42,38 +49,53 @@ files:
 - lib/sitemap_checker.rb
 - lib/sitemap_checker/version.rb
 - sitemap_checker.gemspec
-- spec/fixtures/sitemap_schema.xml
-- spec/fixtures/valid_sitemap.xml
-- spec/fixtures/valid_sitemap.xml.gz
+- spec/fixtures/siteindex.xml
+- spec/fixtures/siteindex.xml.gz
+- spec/fixtures/siteindex.xsd
+- spec/fixtures/sitemap.xml
+- spec/fixtures/sitemap.xml.gz
+- spec/fixtures/sitemap.xsd
 - spec/sitemap_checker_spec.rb
 - spec/spec_helper.rb
 homepage: https://github.com/gerlandop/sitemap_checker
 licenses: []
+
 post_install_message: 
 rdoc_options: []
-require_paths:
+
+require_paths: 
 - lib
-required_ruby_version: !ruby/object:Gem::Requirement
+required_ruby_version: !ruby/object:Gem::Requirement 
   none: false
-  requirements:
-  - - ! '>='
-    - !ruby/object:Gem::Version
-      version: '0'
-required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 2002549777813010636
+      segments: 
+      - 0
+      version: "0"
+required_rubygems_version: !ruby/object:Gem::Requirement 
   none: false
-  requirements:
-  - - ! '>='
-    - !ruby/object:Gem::Version
-      version: '0'
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 2002549777813010636
+      segments: 
+      - 0
+      version: "0"
 requirements: []
+
 rubyforge_project: 
 rubygems_version: 1.8.24
 signing_key: 
 specification_version: 3
 summary: Gets status of Urls in SiteMap
-test_files:
-- spec/fixtures/sitemap_schema.xml
-- spec/fixtures/valid_sitemap.xml
-- spec/fixtures/valid_sitemap.xml.gz
+test_files: 
+- spec/fixtures/siteindex.xml
+- spec/fixtures/siteindex.xml.gz
+- spec/fixtures/siteindex.xsd
+- spec/fixtures/sitemap.xml
+- spec/fixtures/sitemap.xml.gz
+- spec/fixtures/sitemap.xsd
 - spec/sitemap_checker_spec.rb
 - spec/spec_helper.rb