resync 0.1.0

data/spec/unit/resync/resource_dump_manifest_spec.rb

@@ -0,0 +1,59 @@
+require_relative 'shared/base_resource_list_examples'
+
+module Resync
+  describe ResourceDumpManifest do
+    it_behaves_like BaseResourceList
+
+    describe 'links' do
+      it_behaves_like Augmented
+    end
+
+    describe 'XML conversion' do
+      describe '#from_xml' do
+        it 'parses an XML string' do
+          data = File.read('spec/data/examples/example-18.xml')
+          urlset = ResourceDumpManifest.load_from_xml(XML.element(data))
+
+          links = urlset.links
+          expect(links.size).to eq(1)
+          ln0 = links[0]
+          expect(ln0.rel).to eq('up')
+          expect(ln0.uri).to eq(URI('http://example.com/dataset1/capabilitylist.xml'))
+
+          md = urlset.metadata
+          expect(md.capability).to eq('resourcedump-manifest')
+          expect(md.at_time).to be_time(Time.utc(2013, 1, 3, 9))
+          expect(md.completed_time).to be_time(Time.utc(2013, 1, 3, 9, 2))
+
+          urls = urlset.resources
+          expect(urls.size).to eq(2)
+
+          expected_lastmods = [Time.utc(2013, 1, 2, 13), Time.utc(2013, 1, 2, 14)]
+          expected_hashes = [{ 'md5' => '1584abdf8ebdc9802ac0c6a7402c03b6' }, { 'md5' => '1e0d5cb8ef6ba40c99b14c0237be735e', 'sha-256' => '854f61290e2e197a11bc91063afce22e43f8ccc655237050ace766adc68dc784' }]
+          expected_lengths = [8876, 14_599]
+          expected_types = ['text/html', 'application/pdf']
+
+          (0..1).each do |i|
+            url = urls[i]
+            expect(url.uri).to eq(URI("http://example.com/res#{i + 1}"))
+            expect(url.modified_time).to be_time(expected_lastmods[i])
+            md = url.metadata
+            expect(md.hashes).to eq(expected_hashes[i])
+            expect(md.length).to eq(expected_lengths[i])
+            expect(md.mime_type).to be_mime_type(expected_types[i])
+            expect(md.path).to eq("/resources/res#{i + 1}")
+          end
+        end
+      end
+
+      describe '#save_to_xml' do
+        it 'can round-trip to XML' do
+          data = File.read('spec/data/examples/example-18.xml')
+          manifest = ResourceDumpManifest.load_from_xml(XML.element(data))
+          xml = manifest.save_to_xml
+          expect(xml).to be_xml(data)
+        end
+      end
+    end
+  end
+end

data/spec/unit/resync/resource_dump_spec.rb

@@ -0,0 +1,62 @@
+require_relative 'shared/base_resource_list_examples'
+
+module Resync
+  describe ResourceDump do
+    it_behaves_like BaseResourceList
+
+    describe 'links' do
+      it_behaves_like Augmented
+    end
+
+    describe 'XML conversion' do
+      describe '#from_xml' do
+        it 'parses an XML string' do
+          xml = File.read('spec/data/examples/example-17.xml')
+          urlset = ResourceDump.load_from_xml(XML.element(xml))
+          links = urlset.links
+          expect(links.size).to eq(1)
+          ln0 = links[0]
+          expect(ln0.rel).to eq('up')
+          expect(ln0.uri).to eq(URI('http://example.com/dataset1/capabilitylist.xml'))
+
+          md = urlset.metadata
+          expect(md.capability).to eq('resourcedump')
+          expect(md.at_time).to be_time(Time.utc(2013, 1, 3, 9))
+          expect(md.completed_time).to be_time(Time.utc(2013, 1, 3, 9, 4))
+
+          urls = urlset.resources
+          expect(urls.size).to eq(3)
+
+          expected_lengths = [4765, 9875, 2298]
+          expected_ats = [Time.utc(2013, 1, 3, 9), Time.utc(2013, 1, 3, 9, 1), Time.utc(2013, 1, 3, 9, 3)]
+          expected_completeds = [Time.utc(2013, 1, 3, 9, 2), Time.utc(2013, 1, 3, 9, 3), Time.utc(2013, 1, 3, 9, 4)]
+
+          (0..2).each do |i|
+            url = urls[i]
+            expect(url.uri).to eq(URI("http://example.com/resourcedump-part#{i + 1}.zip"))
+            md = url.metadata
+            expect(md.mime_type).to be_mime_type('application/zip')
+            expect(md.length).to eq(expected_lengths[i])
+            expect(md.at_time).to be_time(expected_ats[i])
+            expect(md.completed_time).to be_time(expected_completeds[i])
+            links = url.links
+            expect(links.size).to eq(1)
+            ln = links[0]
+            expect(ln.rel).to eq('contents')
+            expect(ln.uri).to eq(URI("http://example.com/resourcedump_manifest-part#{i + 1}.xml"))
+            expect(ln.mime_type).to be_mime_type('application/xml')
+          end
+        end
+      end
+
+      describe '#save_to_xml' do
+        it 'can round-trip to XML' do
+          data = File.read('spec/data/examples/example-17.xml')
+          dump = ResourceDump.load_from_xml(XML.element(data))
+          xml = dump.save_to_xml
+          expect(xml).to be_xml(data)
+        end
+      end
+    end
+  end
+end

data/spec/unit/resync/resource_list_index_spec.rb

@@ -0,0 +1,53 @@
+require_relative 'shared/base_resource_list_examples'
+
+module Resync
+  describe ResourceListIndex do
+    it_behaves_like BaseResourceList
+
+    describe 'links' do
+      it_behaves_like Augmented
+    end
+
+    describe 'XML conversion' do
+      describe '#from_xml' do
+        it 'parses an XML string' do
+          data = XML.element(File.read('spec/data/examples/example-15.xml'))
+          sitemapindex = ResourceListIndex.load_from_xml(data)
+
+          links = sitemapindex.links
+          expect(links.size).to eq(1)
+          ln0 = links[0]
+          expect(ln0.rel).to eq('up')
+          expect(ln0.uri).to eq(URI('http://example.com/dataset1/capabilitylist.xml'))
+
+          md = sitemapindex.metadata
+          expect(md.capability).to eq('resourcelist')
+          expect(md.at_time).to be_time(Time.utc(2013, 1, 3, 9))
+          expect(md.completed_time).to be_time(Time.utc(2013, 1, 3, 9, 10))
+
+          sitemaps = sitemapindex.resources
+          expect(sitemaps.size).to eq(3)
+
+          expected_times = [Time.utc(2013, 1, 3, 9), Time.utc(2013, 1, 3, 9, 3), Time.utc(2013, 1, 3, 9, 7)]
+          (0..2).each do |i|
+            sitemap = sitemaps[i]
+            expect(sitemap.uri).to eq(URI("http://example.com/resourcelist#{i + 1}.xml"))
+            md = sitemap.metadata
+            expect(md.at_time).to be_time(expected_times[i])
+          end
+        end
+      end
+
+      describe '#save_to_xml' do
+        it 'can round-trip to XML' do
+          data = XML.element(File.read('spec/data/examples/example-15.xml'))
+          list = ResourceListIndex.load_from_xml(XML.element(data), mapping: :sitemapindex)
+          xml = list.save_to_xml
+          expect(xml).to be_xml(data)
+        end
+      end
+
+    end
+
+  end
+end

data/spec/unit/resync/resource_list_spec.rb

@@ -0,0 +1,60 @@
+require_relative 'shared/base_resource_list_examples'
+
+module Resync
+  describe ResourceList do
+    it_behaves_like BaseResourceList
+
+    describe 'links' do
+      it_behaves_like Augmented
+    end
+
+    describe 'XML conversion' do
+      describe '#from_xml' do
+        it 'parses an XML string' do
+          xml = File.read('spec/data/examples/example-16.xml')
+          list = ResourceList.load_from_xml(XML.element(xml))
+          links = list.links
+          expect(links.size).to eq(2)
+          ln0 = links[0]
+          expect(ln0.rel).to eq('up')
+          expect(ln0.uri).to eq(URI('http://example.com/dataset1/capabilitylist.xml'))
+          ln1 = links[1]
+          expect(ln1.rel).to eq('index')
+          expect(ln1.uri).to eq(URI('http://example.com/dataset1/resourcelist-index.xml'))
+
+          md = list.metadata
+          expect(md.capability).to eq('resourcelist')
+          expect(md.at_time).to be_time(Time.utc(2013, 1, 3, 9))
+
+          urls = list.resources
+          expect(urls.size).to eq(2)
+
+          expected_lastmods = [Time.utc(2013, 1, 2, 13), Time.utc(2013, 1, 2, 14)]
+          expected_hashes = [{ 'md5' => '1584abdf8ebdc9802ac0c6a7402c8753' }, { 'md5' => '4556abdf8ebdc9802ac0c6a7402c9881' }]
+          expected_lengths = [4385, 883]
+          expected_types = ['application/pdf', 'image/png']
+
+          (0..1).each do |i|
+            url = urls[i]
+            expect(url.uri).to eq(URI("http://example.com/res#{i + 3}"))
+            expect(url.modified_time).to be_time(expected_lastmods[i])
+            md = url.metadata
+            expect(md.hashes).to eq(expected_hashes[i])
+            expect(md.length).to eq(expected_lengths[i])
+            expect(md.mime_type).to be_mime_type(expected_types[i])
+          end
+        end
+      end
+
+      describe '#save_to_xml' do
+        it 'can round-trip to XML' do
+          data = File.read('spec/data/examples/example-16.xml')
+          list = ResourceList.load_from_xml(XML.element(data))
+          xml = list.save_to_xml
+          expect(xml).to be_xml(data)
+        end
+      end
+    end
+
+  end
+end

data/spec/unit/resync/resource_spec.rb

@@ -0,0 +1,176 @@
+require 'spec_helper'
+require_relative 'shared/uri_field_examples'
+require_relative 'shared/augmented_examples'
+
+module Resync
+  describe Resource do
+    describe '#new' do
+
+      describe 'uri' do
+        it_behaves_like 'a URI field'
+      end
+
+      describe 'modified_time' do
+        it 'accepts a modified_time timestamp' do
+          lastmod = Time.utc(1997, 7, 16, 19, 20, 30.45)
+          resource = Resource.new(uri: 'http://example.org', modified_time: lastmod)
+          expect(resource.modified_time).to be_time(lastmod)
+        end
+
+        it 'defaults to nil if no modified_time timestamp specified' do
+          resource = Resource.new(uri: 'http://example.org')
+          expect(resource.modified_time).to be_nil
+        end
+      end
+
+      describe 'links' do
+        def required_arguments
+          { uri: 'http://example.org' }
+        end
+        it_behaves_like Augmented
+      end
+
+      describe 'metadata' do
+        it 'accepts metadata' do
+          md = Metadata.new
+          resource = Resource.new(uri: 'http://example.org', metadata: md)
+          expect(resource.metadata).to eq(md)
+        end
+
+        it 'defaults to nil if metadata not specified' do
+          resource = Resource.new(uri: 'http://example.org')
+          expect(resource.metadata).to be_nil
+        end
+      end
+
+      describe 'changefreq' do
+        it 'accepts a change frequency' do
+          cf = Types::ChangeFrequency::DAILY
+          resource = Resource.new(uri: 'http://example.org', changefreq: cf)
+          expect(resource.changefreq).to eq(cf)
+        end
+
+        it 'defaults to nil if no change frequency specified' do
+          resource = Resource.new(uri: 'http://example.org')
+          expect(resource.changefreq).to be_nil
+        end
+      end
+
+      describe 'priority' do
+        it 'accepts a priority' do
+          priority = 1.234
+          resource = Resource.new(uri: 'http://example.org', priority: priority)
+          expect(resource.priority).to eq(priority)
+        end
+
+        it 'defaults to nil if no priority specified' do
+          resource = Resource.new(uri: 'http://example.org')
+          expect(resource.priority).to be_nil
+        end
+      end
+    end
+
+    describe 'capability' do
+      it 'extracts the capability from the metadata' do
+        md = Metadata.new(capability: 'changelist')
+        resource = Resource.new(uri: 'http://example.org', metadata: md)
+        expect(resource.capability).to eq('changelist')
+      end
+
+      it 'returns nil if no metadata was specified' do
+        resource = Resource.new(uri: 'http://example.org')
+        expect(resource.capability).to be_nil
+      end
+    end
+
+    describe 'XML conversion' do
+      describe '#from_xml' do
+        it 'parses an XML string' do
+          xml = '<url xmlns="http://www.sitemaps.org/schemas/sitemap/0.9" xmlns:rs="http://www.openarchives.org/rs/terms/">
+                    <loc>http://example.com/res1</loc>
+                    <lastmod>2013-01-03T18:00:00Z</lastmod>
+                    <rs:md change="updated"
+                           hash="md5:1584abdf8ebdc9802ac0c6a7402c03b6"
+                           length="8876"
+                           type="text/html"/>
+                    <rs:ln rel="duplicate"
+                           pri="1"
+                           href="http://mirror1.example.com/res1"
+                           modified="2013-01-03T18:00:00Z"/>
+                    <rs:ln rel="duplicate"
+                           pri="2"
+                           href="http://mirror2.example.com/res1"
+                           modified="2013-01-03T18:00:00Z"/>
+                    <rs:ln rel="duplicate"
+                           pri="3"
+                           href="gsiftp://gridftp.example.com/res1"
+                           modified="2013-01-03T18:00:00Z"/>
+                </url>'
+          resource = Resource.load_from_xml(XML.element(xml))
+          expect(resource).to be_a(Resource)
+          expect(resource.uri).to eq(URI('http://example.com/res1'))
+          expect(resource.modified_time).to be_time(Time.utc(2013, 1, 3, 18))
+          metadata = resource.metadata
+          expect(metadata.change).to eq(Types::Change::UPDATED)
+          expect(metadata.hash('md5')).to eq('1584abdf8ebdc9802ac0c6a7402c03b6')
+          expect(metadata.length).to eq(8_876)
+          expect(metadata.mime_type).to be_mime_type('text/html')
+          links = resource.links
+          expect(links.size).to eq(3)
+          expected_uris = [URI('http://mirror1.example.com/res1'),
+                           URI('http://mirror2.example.com/res1'),
+                           URI('gsiftp://gridftp.example.com/res1')]
+
+          (0..2).each do |i|
+            ln = links[i]
+            expect(ln.rel).to eq('duplicate')
+            expect(ln.priority).to eq(i + 1)
+            expect(ln.uri).to eq(expected_uris[i])
+            expect(ln.modified_time).to be_time(Time.utc(2013, 1, 3, 18))
+          end
+        end
+      end
+
+      it 'can round-trip to XML' do
+        data = '<url xmlns="http://www.sitemaps.org/schemas/sitemap/0.9" xmlns:rs="http://www.openarchives.org/rs/terms/">
+                    <loc>http://example.com/res1</loc>
+                    <lastmod>2013-01-03T18:00:00Z</lastmod>
+                    <changefreq>daily</changefreq>
+                    <rs:ln rel="duplicate"
+                           pri="1"
+                           href="http://mirror1.example.com/res1"
+                           modified="2013-01-03T18:00:00Z"/>
+                    <rs:ln rel="duplicate"
+                           pri="2"
+                           href="http://mirror2.example.com/res1"
+                           modified="2013-01-03T18:00:00Z"/>
+                    <rs:md change="updated"
+                           hash="md5:1584abdf8ebdc9802ac0c6a7402c03b6"
+                           length="8876"
+                           type="text/html"/>
+                    <rs:ln rel="duplicate"
+                           pri="3"
+                           href="gsiftp://gridftp.example.com/res1"
+                           modified="2013-01-03T18:00:00Z"/>
+                </url>'
+        resource = Resource.load_from_xml(XML.element(data))
+
+        # Since resource isn't a root element, these won't be hacked in as in BaseResourceList#pre_save()
+        xml = resource.save_to_xml
+        xml.add_namespace('http://www.sitemaps.org/schemas/sitemap/0.9')
+        xml.add_namespace('rs', 'http://www.openarchives.org/rs/terms/')
+
+        expect(xml).to be_xml(data)
+      end
+
+      it 'doesn\'t have side effects that prevent <sitemapindex> parsing' do
+        Resource.xml_mapping_nodes(mapping: :sitemapindex)
+
+        data = File.read('spec/data/examples/example-8.xml')
+        root = REXML::Document.new(data).root
+        sitemapindex = XMLParser.parse(root)
+        expect(sitemapindex).to be_a(ResourceListIndex)
+      end
+    end
+  end
+end

data/spec/unit/resync/shared/augmented_examples.rb

@@ -0,0 +1,58 @@
+module Resync
+  # TODO: Figure out if 'up' is mandatory for all resource lists, or just CapabilityList
+  # TODO: Consider requiring this in base_resource_list_examples again, if 'up' is mandatory
+  RSpec.shared_examples Augmented do
+
+    # TODO: Find a better way to express this
+    def new_instance(**args)
+      required_args = (defined? required_arguments) ? required_arguments : {}
+      args = required_args.merge(args)
+      described_class.new(**args)
+    end
+
+    describe 'links' do
+      it 'accepts a list of links' do
+        links = [Link.new(rel: 'describedby', uri: 'http://example.org/'), Link.new(rel: 'duplicate', uri: 'http://example.com/')]
+        list = new_instance(links: links)
+        expect(list.links).to eq(links)
+      end
+
+      it 'defaults to an empty list if no links are specified' do
+        list = new_instance
+        expect(list.links).to eq([])
+      end
+
+    end
+
+    describe '#links_for' do
+      it 'can retrieve a list of links by rel' do
+        links = [
+          Link.new(rel: 'describedby', uri: 'http://example.org/desc1'),
+          Link.new(rel: 'duplicate', uri: 'http://example.com/dup1'),
+          Link.new(rel: 'describedby', uri: 'http://example.org/desc2'),
+          Link.new(rel: 'duplicate', uri: 'http://example.com/dup2')
+        ]
+        list = new_instance(links: links)
+        expect(list.links_for(rel: 'describedby')).to eq([links[0], links[2]])
+        expect(list.links_for(rel: 'duplicate')).to eq([links[1], links[3]])
+        expect(list.links_for(rel: 'elvis')).to eq([])
+      end
+    end
+
+    describe '#link_for' do
+      it 'can retrieve the first link for a rel' do
+        links = [
+          Link.new(rel: 'describedby', uri: 'http://example.org/desc1'),
+          Link.new(rel: 'duplicate', uri: 'http://example.com/dup1'),
+          Link.new(rel: 'describedby', uri: 'http://example.org/desc2'),
+          Link.new(rel: 'duplicate', uri: 'http://example.com/dup2')
+        ]
+        list = new_instance(links: links)
+        expect(list.link_for(rel: 'describedby')).to eq(links[0])
+        expect(list.link_for(rel: 'duplicate')).to eq(links[1])
+        expect(list.link_for(rel: 'elvis')).to eq(nil)
+      end
+    end
+
+  end
+end

data/spec/unit/resync/shared/base_resource_list_examples.rb

@@ -0,0 +1,103 @@
+require 'spec_helper'
+require_relative 'augmented_examples'
+
+module Resync
+  RSpec.shared_examples BaseResourceList do
+
+    # ------------------------------------------------------
+    # "Virtual" fixture methods
+
+    # TODO: Find a better way to express this
+    def resource_list
+      (defined? valid_resources) ? valid_resources : [Resource.new(uri: 'http://example.org/'), Resource.new(uri: 'http://example.com/')]
+    end
+
+    # TODO: Find a better way to express this
+    def new_instance(**args)
+      required_args = (defined? required_arguments) ? required_arguments : {}
+      args = required_args.merge(args)
+      described_class.new(**args)
+    end
+
+    # ------------------------------------------------------
+    # Tests
+
+    describe '#new' do
+      describe 'resources' do
+        it 'accepts a list of resources' do
+          puts defined? valid_resources
+          resources = resource_list
+          list = new_instance(resources: resources)
+          expect(list.resources).to eq(resources)
+        end
+
+        it 'defaults to an empty list if no resources are specified' do
+          list = new_instance
+          expect(list.resources).to eq([])
+        end
+      end
+
+      describe 'metadata' do
+        it 'accepts metadata' do
+          metadata = Metadata.new(capability: described_class::CAPABILITY)
+          list = new_instance(metadata: metadata)
+          expect(list.metadata).to eq(metadata)
+        end
+
+        it 'defaults (otherwise empty) metadata with capability CAPABILITY if no metadata specified' do
+          list = new_instance
+          metadata = list.metadata
+          expect(metadata.capability).to eq(described_class::CAPABILITY)
+        end
+
+        it 'fails if metadata does not have capability CAPABILITY' do
+          expect { new_instance(metadata: Metadata.new) }.to raise_error(ArgumentError)
+          expect { new_instance(metadata: Metadata.new(capability: "not_#{described_class::CAPABILITY}")) }.to raise_error(ArgumentError)
+        end
+      end
+    end
+
+    describe '#resources_for' do
+      it 'can retrieve a list of resources by capability' do
+        unless defined? valid_resources
+          resources = [
+            Resource.new(uri: 'http://example.com/dataset1/resourcedump1.xml', metadata: Metadata.new(capability: 'resourcedump')),
+            Resource.new(uri: 'http://example.com/dataset1/changelist1.xml', metadata: Metadata.new(capability: 'changelist')),
+            Resource.new(uri: 'http://example.com/dataset1/resourcedump2.xml', metadata: Metadata.new(capability: 'resourcedump')),
+            Resource.new(uri: 'http://example.com/dataset1/changelist2.xml', metadata: Metadata.new(capability: 'changelist'))
+          ]
+          list = new_instance(resources: resources)
+          expect(list.resources_for(capability: 'resourcedump')).to eq([resources[0], resources[2]])
+          expect(list.resources_for(capability: 'changelist')).to eq([resources[1], resources[3]])
+          expect(list.resources_for(capability: 'changedump')).to eq([])
+        end
+      end
+    end
+
+    describe '#resource_for' do
+      it 'can retrieve the first resource for a capability' do
+        unless defined? valid_resources
+          resources = [
+            Resource.new(uri: 'http://example.com/dataset1/resourcedump1.xml', metadata: Metadata.new(capability: 'resourcedump')),
+            Resource.new(uri: 'http://example.com/dataset1/changelist1.xml', metadata: Metadata.new(capability: 'changelist')),
+            Resource.new(uri: 'http://example.com/dataset1/resourcedump2.xml', metadata: Metadata.new(capability: 'resourcedump')),
+            Resource.new(uri: 'http://example.com/dataset1/changelist2.xml', metadata: Metadata.new(capability: 'changelist'))
+          ]
+          list = new_instance(resources: resources)
+          expect(list.resource_for(capability: 'resourcedump')).to eq(resources[0])
+          expect(list.resource_for(capability: 'changelist')).to eq(resources[1])
+          expect(list.resource_for(capability: 'changedump')).to eq(nil)
+        end
+      end
+    end
+
+    describe 'capability' do
+      it 'extracts the capability' do
+        metadata = Metadata.new(capability: described_class::CAPABILITY)
+        list = new_instance(metadata: metadata)
+        expect(list.capability).to eq(described_class::CAPABILITY)
+      end
+    end
+
+  end
+end