resync 0.1.0

data/spec/data/examples/example-32.xml

@@ -0,0 +1,22 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9"
+        xmlns:rs="http://www.openarchives.org/rs/terms/">
+  <rs:ln rel="up"
+         href="http://aggregator1.example.com/dataset1/capabilitylist.xml"/>
+  <rs:md capability="changelist"
+         from="2013-01-03T11:00:00Z"/>
+  <url>
+    <loc>http://aggregator1.example.com/res1.html</loc>
+    <lastmod>2013-01-03T20:00:00Z</lastmod>
+    <rs:md hash="md5:1584abdf8ebdc9802ac0c6a7402c03b6"
+           length="8876"
+           type="text/html"
+           change="updated"/>
+    <rs:ln rel="via"
+           href="http://original.example.com/res1.html"
+           modified="2013-01-03T07:00:00Z"
+           hash="md5:1584abdf8ebdc9802ac0c6a7402c03b6"
+           length="8876"
+           type="text/html"/>
+  </url>
+</urlset>

data/spec/data/examples/example-33.xml

@@ -0,0 +1,22 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9"
+        xmlns:rs="http://www.openarchives.org/rs/terms/">
+  <rs:ln rel="up"
+         href="http://aggregator2.example.com/dataset1/capabilitylist.xml"/>
+  <rs:md capability="changelist"
+         from="2013-01-03T12:00:00Z"/>
+  <url>
+    <loc>http://aggregator2.example.com/res1.html</loc>
+    <lastmod>2013-01-04T09:00:00Z</lastmod>
+    <rs:md hash="md5:1584abdf8ebdc9802ac0c6a7402c03b6"
+           length="8876"
+           type="text/html"
+           change="updated"/>
+    <rs:ln rel="via"
+           href="http://original.example.com/res1.html"
+           modified="2013-01-03T07:00:00Z"
+           hash="md5:1584abdf8ebdc9802ac0c6a7402c03b6"
+           length="8876"
+           type="text/html"/>
+  </url>
+</urlset>

data/spec/data/examples/example-4.xml

@@ -0,0 +1,10 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9"
+        xmlns:rs="http://www.openarchives.org/rs/terms/">
+  <rs:md capability="resourcedump"
+         at="2013-01-03T09:00:00Z"/>
+  <url>
+    <loc>http://example.com/resourcedump.zip</loc>
+    <lastmod>2013-01-03T09:00:00Z</lastmod>
+  </url>
+</urlset>

data/spec/data/examples/example-5.xml

@@ -0,0 +1,18 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9"
+        xmlns:rs="http://www.openarchives.org/rs/terms/">
+  <rs:md capability="resourcedump-manifest"
+         at="2013-01-03T09:00:00Z"/>
+  <url>
+    <loc>http://example.com/res1</loc>
+    <lastmod>2013-01-03T03:00:00Z</lastmod>
+    <rs:md hash="md5:1584abdf8ebdc9802ac0c6a7402c03b6"
+           path="/resources/res1"/>
+  </url>
+  <url>
+    <loc>http://example.com/res2</loc>
+    <lastmod>2013-01-03T04:00:00Z</lastmod>
+    <rs:md hash="md5:1e0d5cb8ef6ba40c99b14c0237be735e"
+           path="/resources/res2"/>
+  </url>
+</urlset>

data/spec/data/examples/example-6.xml

@@ -0,0 +1,21 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9"
+        xmlns:rs="http://www.openarchives.org/rs/terms/">
+  <rs:ln rel="describedby"
+         href="http://example.com/info_about_set1_of_resources.xml"/>
+  <rs:ln rel="up"
+         href="http://example.com/resourcesync_description.xml"/>
+  <rs:md capability="capabilitylist"/>
+  <url>
+    <loc>http://example.com/dataset1/resourcelist.xml</loc>
+    <rs:md capability="resourcelist"/>
+  </url>
+  <url>
+    <loc>http://example.com/dataset1/resourcedump.xml</loc>
+    <rs:md capability="resourcedump"/>
+  </url>
+  <url>
+    <loc>http://example.com/dataset1/changelist.xml</loc>
+    <rs:md capability="changelist"/>
+  </url>
+</urlset>

data/spec/data/examples/example-7.xml

@@ -0,0 +1,13 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9"
+        xmlns:rs="http://www.openarchives.org/rs/terms/">
+  <rs:ln rel="describedby"
+         href="http://example.com/info-about-source.xml"/>
+  <rs:md capability="description"/>
+  <url>
+    <loc>http://example.com/dataset1/capabilitylist.xml</loc>
+    <rs:md capability="capabilitylist"/>
+    <rs:ln rel="describedby"
+           href="http://example.com/info_about_set1_of_resources.xml"/>
+  </url>
+</urlset>

data/spec/data/examples/example-8.xml

@@ -0,0 +1,12 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<sitemapindex xmlns="http://www.sitemaps.org/schemas/sitemap/0.9"
+              xmlns:rs="http://www.openarchives.org/rs/terms/">
+  <rs:md capability="resourcelist"
+         at="2013-01-03T09:00:00Z"/>
+  <sitemap>
+    <loc>http://example.com/resourcelist-part1.xml</loc>
+  </sitemap>
+  <sitemap>
+    <loc>http://example.com/resourcelist-part2.xml</loc>
+  </sitemap>
+</sitemapindex>

data/spec/data/resourcesync.xsd

@@ -0,0 +1,148 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<xsd:schema xmlns:xsd="http://www.w3.org/2001/XMLSchema"
+            targetNamespace="http://www.openarchives.org/rs/terms/"
+            xmlns="http://www.openarchives.org/rs/terms/"
+            elementFormDefault="qualified">
+
+<xsd:annotation>
+<xsd:documentation>
+XML Schema for ResoureSync extensions to Sitemap files.
+See: http://www.openarchives.org/rs/ .
+2013-01-06 first stab [Simeon Warner],
+2013-08-15 for v0.9.1 [Simeon Warner].
+</xsd:documentation>
+</xsd:annotation>
+
+  <xsd:element name="md">
+    <xsd:annotation>
+      <xsd:documentation>
+	Metadata about a resource
+      </xsd:documentation>
+    </xsd:annotation>
+    <xsd:complexType>
+      <xsd:complexContent>
+        <xsd:restriction base="xsd:anyType">
+          <xsd:attribute name="at" type="xsd:dateTime"/>
+          <xsd:attribute name="capability" type="xsd:string"/>
+          <xsd:attribute name="change" type="changeType"/>
+          <xsd:attribute name="completed" type="xsd:dateTime"/>
+          <xsd:attribute name="encoding" type="xsd:string"/>
+          <xsd:attribute name="from" type="xsd:dateTime"/>
+          <xsd:attribute name="hash" type="xsd:string"/>
+          <xsd:attribute name="length" type="xsd:integer"/>
+          <xsd:attribute name="modified" type="xsd:dateTime"/>
+          <xsd:attribute name="path" type="pathType"/>
+          <xsd:attribute name="type" type="contentType"/>
+          <xsd:attribute name="until" type="xsd:dateTime"/>
+	  <xsd:anyAttribute namespace="##other" processContents="lax"/>
+        </xsd:restriction>
+      </xsd:complexContent>
+    </xsd:complexType>
+  </xsd:element>
+
+  <xsd:element name="ln">
+    <xsd:annotation>
+      <xsd:documentation>
+	Links to related resources
+      </xsd:documentation>
+    </xsd:annotation>
+    <xsd:complexType>
+      <xsd:complexContent>
+        <xsd:restriction base="xsd:anyType">
+          <xsd:attribute name="encoding" type="xsd:string"/>
+          <xsd:attribute name="hash" type="xsd:string"/>
+          <xsd:attribute name="href" type="xsd:anyURI"/>
+          <xsd:attribute name="length" type="xsd:nonNegativeInteger"/>
+          <xsd:attribute name="modified" type="xsd:dateTime"/>
+          <xsd:attribute name="path" type="pathType"/>
+          <xsd:attribute name="pri" type="priType"/>
+          <xsd:attribute name="rel" type="relType"/>
+          <xsd:attribute name="type" type="contentType"/>
+	  <xsd:anyAttribute namespace="##other" processContents="lax"/>
+        </xsd:restriction>
+      </xsd:complexContent>
+    </xsd:complexType>
+  </xsd:element>
+
+  <xsd:simpleType name="changeType">
+    <xsd:annotation>
+      <xsd:documentation>
+        Allowed values of the change attribute are
+	created, updated, deleted
+      </xsd:documentation>
+    </xsd:annotation>
+    <xsd:restriction base="xsd:string">
+      <xsd:enumeration value="created"/>
+      <xsd:enumeration value="updated"/>
+      <xsd:enumeration value="deleted"/>
+    </xsd:restriction>
+  </xsd:simpleType>
+
+  <xsd:simpleType name="pathType">
+    <xsd:annotation>
+      <xsd:documentation>
+        Path values must start with a slash, must not end with a slash
+      </xsd:documentation>
+    </xsd:annotation>
+    <xsd:restriction base="xsd:string">
+      <xsd:pattern value="/.*[^/]"/>
+    </xsd:restriction>
+  </xsd:simpleType>
+
+  <xsd:simpleType name="contentType">
+    <xsd:annotation>
+      <xsd:documentation>
+        MIME Content Types are described in RFCs 2045,2046
+        http://tools.ietf.org/html/rfc2045
+        http://tools.ietf.org/html/rfc2046
+      </xsd:documentation>
+    </xsd:annotation>
+    <xsd:restriction base="xsd:string">
+      <xsd:pattern value="[\w-]+/[\w-]+"/>
+    </xsd:restriction>
+  </xsd:simpleType>
+
+  <xsd:simpleType name="priType">
+    <xsd:annotation>
+      <xsd:documentation>
+        The priority attribute may have values 1 through 999999
+      </xsd:documentation>
+    </xsd:annotation>
+    <xsd:restriction base="xsd:integer">
+      <xsd:minInclusive value="1"/>
+      <xsd:maxInclusive value="999999"/>
+    </xsd:restriction>
+  </xsd:simpleType>
+
+  <xsd:simpleType name="relType">
+    <xsd:annotation>
+      <xsd:documentation>
+	Syntax for link relation values is specified by
+	http://tools.ietf.org/html/rfc5988 and the relevant
+	portion is:
+
+	# link-param     = ( ( "rel" "=" relation-types )
+	#	           ... 
+	# relation-type  = reg-rel-type | ext-rel-type 
+	# reg-rel-type   = LOALPHA *( LOALPHA | DIGIT | "." | "-" ) 
+	# ext-rel-type   = URI
+	#
+
+	Registered link relations (reg-rel-type) are listed at
+	http://www.iana.org/assignments/link-relations/link-relations.xhtml
+	and URIs are allowed for extension (ext-rel-type)
+      </xsd:documentation>
+    </xsd:annotation>
+    <xsd:union>
+      <xsd:simpleType>
+	<xsd:restriction base="xsd:string">
+          <xsd:pattern value="[a-z][a-z0-9\.\-]*"/>
+        </xsd:restriction>
+      </xsd:simpleType>
+      <xsd:simpleType>
+	<xsd:restriction base="xsd:anyURI"/>
+      </xsd:simpleType>
+    </xsd:union>
+  </xsd:simpleType>
+
+</xsd:schema>

data/spec/data/siteindex.xsd

@@ -0,0 +1,75 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<xsd:schema xmlns:xsd="http://www.w3.org/2001/XMLSchema"
+            targetNamespace="http://www.sitemaps.org/schemas/sitemap/0.9"
+            xmlns="http://www.sitemaps.org/schemas/sitemap/0.9"
+            elementFormDefault="qualified">
+  <xsd:annotation>
+    <xsd:documentation>
+      XML Schema for Sitemap index files.
+      Last Modifed 2009-04-08
+    </xsd:documentation>
+  </xsd:annotation>
+
+  <xsd:element name="sitemapindex">
+    <xsd:annotation>
+      <xsd:documentation>
+        Container for a set of up to 50,000 sitemap URLs.
+        This is the root element of the XML file.
+      </xsd:documentation>
+    </xsd:annotation>
+    <xsd:complexType>
+      <xsd:sequence>
+        <xsd:any namespace="##other" minOccurs="0" maxOccurs="unbounded" processContents="strict"/>
+        <xsd:element name="sitemap" type="tSitemap" maxOccurs="unbounded"/>
+      </xsd:sequence>
+    </xsd:complexType>
+  </xsd:element>
+
+  <xsd:complexType name="tSitemap">
+    <xsd:annotation>
+      <xsd:documentation>
+        Container for the data needed to describe a sitemap.
+      </xsd:documentation>
+    </xsd:annotation>
+    <xsd:sequence>
+      <xsd:element name="loc" type="tLocSitemap"/>
+      <xsd:element name="lastmod" type="tLastmodSitemap" minOccurs="0"/>
+      <xsd:any namespace="##other" minOccurs="0" maxOccurs="unbounded" processContents="strict"/>
+    </xsd:sequence>
+  </xsd:complexType>
+
+  <xsd:simpleType name="tLocSitemap">
+    <xsd:annotation>
+      <xsd:documentation>
+        REQUIRED: The location URI of a sitemap.
+        The URI must conform to RFC 2396 (http://www.ietf.org/rfc/rfc2396.txt).
+      </xsd:documentation>
+    </xsd:annotation>
+    <xsd:restriction base="xsd:anyURI">
+      <xsd:minLength value="12"/>
+      <xsd:maxLength value="2048"/>
+    </xsd:restriction>
+  </xsd:simpleType>
+
+  <xsd:simpleType name="tLastmodSitemap">
+    <xsd:annotation>
+      <xsd:documentation>
+        OPTIONAL: The date the document was last modified. The date must conform
+        to the W3C DATETIME format (http://www.w3.org/TR/NOTE-datetime).
+        Example: 2005-05-10
+        Lastmod may also contain a timestamp.
+        Example: 2005-05-10T17:33:30+08:00
+      </xsd:documentation>
+    </xsd:annotation>
+    <xsd:union>
+      <xsd:simpleType>
+        <xsd:restriction base="xsd:date"/>
+      </xsd:simpleType>
+      <xsd:simpleType>
+        <xsd:restriction base="xsd:dateTime"/>
+      </xsd:simpleType>
+    </xsd:union>
+  </xsd:simpleType>
+
+
+</xsd:schema>

data/spec/data/sitemap.xsd

@@ -0,0 +1,116 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<xsd:schema xmlns:xsd="http://www.w3.org/2001/XMLSchema"
+            targetNamespace="http://www.sitemaps.org/schemas/sitemap/0.9"
+            xmlns="http://www.sitemaps.org/schemas/sitemap/0.9"
+            elementFormDefault="qualified">
+  <xsd:annotation>
+    <xsd:documentation>
+      XML Schema for Sitemap files.
+      Last Modifed 2008-03-26
+    </xsd:documentation>
+  </xsd:annotation>
+
+  <xsd:element name="urlset">
+    <xsd:annotation>
+      <xsd:documentation>
+        Container for a set of up to 50,000 document elements.
+        This is the root element of the XML file.
+      </xsd:documentation>
+    </xsd:annotation>
+    <xsd:complexType>
+      <xsd:sequence>
+        <xsd:any namespace="##other" minOccurs="0" maxOccurs="unbounded" processContents="strict"/>
+        <xsd:element name="url" type="tUrl" maxOccurs="unbounded"/>
+      </xsd:sequence>
+    </xsd:complexType>
+  </xsd:element>
+
+  <xsd:complexType name="tUrl">
+    <xsd:annotation>
+      <xsd:documentation>
+        Container for the data needed to describe a document to crawl.
+      </xsd:documentation>
+    </xsd:annotation>
+    <xsd:sequence>
+      <xsd:element name="loc" type="tLoc"/>
+      <xsd:element name="lastmod" type="tLastmod" minOccurs="0"/>
+      <xsd:element name="changefreq" type="tChangeFreq" minOccurs="0"/>
+      <xsd:element name="priority" type="tPriority" minOccurs="0"/>
+      <xsd:any namespace="##other" minOccurs="0" maxOccurs="unbounded" processContents="strict"/>
+    </xsd:sequence>
+  </xsd:complexType>
+
+  <xsd:simpleType name="tLoc">
+    <xsd:annotation>
+      <xsd:documentation>
+        REQUIRED: The location URI of a document.
+        The URI must conform to RFC 2396 (http://www.ietf.org/rfc/rfc2396.txt).
+      </xsd:documentation>
+    </xsd:annotation>
+    <xsd:restriction base="xsd:anyURI">
+      <xsd:minLength value="12"/>
+      <xsd:maxLength value="2048"/>
+    </xsd:restriction>
+  </xsd:simpleType>
+
+  <xsd:simpleType name="tLastmod">
+    <xsd:annotation>
+      <xsd:documentation>
+        OPTIONAL: The date the document was last modified. The date must conform
+        to the W3C DATETIME format (http://www.w3.org/TR/NOTE-datetime).
+        Example: 2005-05-10
+        Lastmod may also contain a timestamp.
+        Example: 2005-05-10T17:33:30+08:00
+      </xsd:documentation>
+    </xsd:annotation>
+    <xsd:union>
+      <xsd:simpleType>
+        <xsd:restriction base="xsd:date"/>
+      </xsd:simpleType>
+      <xsd:simpleType>
+        <xsd:restriction base="xsd:dateTime"/>
+      </xsd:simpleType>
+    </xsd:union>
+  </xsd:simpleType>
+
+  <xsd:simpleType name="tChangeFreq">
+    <xsd:annotation>
+      <xsd:documentation>
+        OPTIONAL: Indicates how frequently the content at a particular URL is
+        likely to change. The value "always" should be used to describe
+        documents that change each time they are accessed. The value "never"
+        should be used to describe archived URLs. Please note that web
+        crawlers may not necessarily crawl pages marked "always" more often.
+        Consider this element as a friendly suggestion and not a command.
+      </xsd:documentation>
+    </xsd:annotation>
+    <xsd:restriction base="xsd:string">
+      <xsd:enumeration value="always"/>
+      <xsd:enumeration value="hourly"/>
+      <xsd:enumeration value="daily"/>
+      <xsd:enumeration value="weekly"/>
+      <xsd:enumeration value="monthly"/>
+      <xsd:enumeration value="yearly"/>
+      <xsd:enumeration value="never"/>
+    </xsd:restriction>
+  </xsd:simpleType>
+
+  <xsd:simpleType name="tPriority">
+    <xsd:annotation>
+      <xsd:documentation>
+        OPTIONAL: The priority of a particular URL relative to other pages
+        on the same site. The value for this element is a number between
+        0.0 and 1.0 where 0.0 identifies the lowest priority page(s).
+        The default priority of a page is 0.5. Priority is used to select
+        between pages on your site. Setting a priority of 1.0 for all URLs
+        will not help you, as the relative priority of pages on your site
+        is what will be considered.
+      </xsd:documentation>
+    </xsd:annotation>
+    <xsd:restriction base="xsd:decimal">
+      <xsd:minInclusive value="0.0"/>
+      <xsd:maxInclusive value="1.0"/>
+    </xsd:restriction>
+  </xsd:simpleType>
+
+</xsd:schema>

data/spec/rspec_custom_matchers.rb

@@ -0,0 +1,89 @@
+require 'rspec/expectations'
+require 'equivalent-xml'
+
+RSpec::Matchers.define :be_xml do |expected|
+
+  def to_nokogiri(xml)
+    case xml
+    when Nokogiri::XML::Element
+      xml
+    when Nokogiri::XML::Document
+      xml.root
+    when String
+      to_nokogiri(Nokogiri::XML(xml))
+    when REXML::Element
+      to_nokogiri(xml.to_s)
+    end
+  end
+
+  def to_pretty(nokogiri)
+    return nil unless nokogiri
+    out = StringIO.new
+    save_options = Nokogiri::XML::Node::SaveOptions::FORMAT | Nokogiri::XML::Node::SaveOptions::NO_DECLARATION
+    nokogiri.write_xml_to(out, encoding: 'UTF-8', indent: 2, save_with: save_options)
+    out.string
+  end
+
+  match do |actual|
+    expected_xml = to_nokogiri(expected) || fail("expected value #{expected} does not appear to be XML")
+    actual_xml = to_nokogiri(actual)
+
+    EquivalentXml.equivalent?(expected_xml, actual_xml, element_order: false, normalize_whitespace: true)
+  end
+
+  failure_message do |actual|
+    expected_string = to_pretty(to_nokogiri(expected))
+    actual_string = to_pretty(to_nokogiri(actual)) || actual
+    "expected XML:\n#{expected_string}\n\nbut was:\n#{actual_string}"
+  end
+
+  failure_message_when_negated do |actual|
+    actual_xml = to_element(actual) || actual
+    "expected not to get XML:\n\t#{actual_xml}"
+  end
+end
+
+RSpec::Matchers.define :be_time do |expected|
+
+  def to_string(time)
+    time.is_a?(Time) ? time.utc.round(2).iso8601(2) : time.to_s
+  end
+
+  match do |actual|
+    if expected
+      fail "Expected value #{expected} is not a Time" unless expected.is_a?(Time)
+      actual.is_a?(Time) && (to_string(expected) == to_string(actual))
+    else
+      return actual.nil?
+    end
+  end
+
+  failure_message do |actual|
+    expected_str = to_string(expected)
+    actual_str = to_string(actual)
+    "expected time:\n#{expected_str}\n\nbut was:\n#{actual_str}"
+  end
+end
+
+def to_mime_type(mime_type)
+  return nil unless mime_type
+  return mime_type if mime_type.is_a?(MIME::Type)
+
+  mt = MIME::Types[mime_type].first
+  return mt if mt
+
+  MIME::Type.new(mime_type)
+end
+
+RSpec::Matchers.define :be_mime_type do |expected|
+
+  expected_mime_type = to_mime_type(expected)
+
+  match do |actual|
+    actual == expected_mime_type
+  end
+
+  failure_message do |actual|
+    "expected MIME type:\n#{expected_mime_type}\nbut was:\n#{actual}"
+  end
+end

data/spec/spec_helper.rb

@@ -0,0 +1,31 @@
+# ------------------------------------------------------------
+# SimpleCov setup
+
+if ENV['COVERAGE']
+  require 'simplecov'
+  require 'simplecov-console'
+
+  SimpleCov.minimum_coverage 100
+  SimpleCov.start do
+    add_filter '/spec/'
+    SimpleCov.formatter = SimpleCov::Formatter::MultiFormatter[
+        SimpleCov::Formatter::HTMLFormatter,
+        SimpleCov::Formatter::Console,
+    ]
+  end
+end
+
+# ------------------------------------------------------------
+# Rspec configuration
+
+RSpec.configure do |config|
+  config.raise_errors_for_deprecations!
+  config.mock_with :rspec
+end
+
+require 'rspec_custom_matchers'
+
+# ------------------------------------------------------------
+# Stash::Harvester
+
+require 'resync'

data/spec/todo.rb

@@ -0,0 +1,11 @@
+require 'rspec/core'
+require 'resync'
+
+# List of TODO items in spec form
+describe Resync do
+
+  describe 'library' do
+    it 'enforces the required/forbidden time attribute table in appendix a of the spec'
+  end
+
+end