resync-client 0.2.4 → 0.2.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 8314c851c5631164009f9a2d88d0c3d403cc70ad
-  data.tar.gz: 533203c8b735acb0c29063efa8112feb80921e45
+  metadata.gz: 6dc391cfdfcafd5a808a5a4736daf19a1e52d418
+  data.tar.gz: 2f983d3560d4bf3390faaeda241e5b6a570be8f9
 SHA512:
-  metadata.gz: 2be76b5aaf5d2141a009ef77336ebb1d78877971d94a1069e369bb25e97251726b548898f3906e9852097e53cde6ec28f68638bebfd14c90f6c124bddbcd14d9
-  data.tar.gz: 0071b77513fc26fee16832f7885e414801e65c4895df3911c4093b5fff0c187c478209605447db5cf9f27eb97a4ae5e457bb093f436eed3bf7cce0c03fe5439c
+  metadata.gz: 65e5d83731a97afab8e3d1d6cbcca7ef716f7acf2e3e7d1ef490763dff15c8daef6539591bd51069b11ddae2f530bcd005928e92b57fdd0805b092e66da7b961
+  data.tar.gz: 468e546e76abb8f29ca6f6a968efd17ab46f785fd082148f0b9b343fca139a69cc2f9dcbd0f491ad37834dab454d104fae35edca5f0e055fcbcba0f34558effc

data/CHANGES.md

@@ -1,3 +1,7 @@
+# 0.2.5
+
+- Added `#all_changes` to transparently download and flatten changes in `ChangeListIndex` and `ChangeDumpIndex` documents, with filtering by time and type
+
 # 0.2.4
 
 - Added `#all_resources` to transparently download and flatten lists in `ChangeListIndex` and `ResourceListIndex`

data/lib/resync/client/mixins/change_index.rb

@@ -0,0 +1,37 @@
+require 'resync'
+require_relative 'resource_client_delegate'
+
+module Resync
+  class Client
+    module Mixins
+      # A resource container whose resources are lists of changes.
+      module ChangeIndex
+        prepend ResourceClientDelegate
+
+        # Downloads and parses each resource list and returns a flattened enumeration
+        # of all changes in each contained list, filtering by date/time, change type,
+        # or both. (Each contained list is only downloaded as needed, and only downloaded
+        # once.) The lists of lists are filtered by +from_time+ and +until_time+, in
+        # non-strict mode (only excluding those lists provably not in the range, i.e.,
+        # including lists without +from_time+ or +until_time+); the individual changes
+        # are filtered by +modified_time+.
+        #
+        # @param of_type [Resync::Types::Change, nil] the change type
+        # @param in_range [Range<Time>, nil] the time range
+        # @return [Enumerator::Lazy<Resync::Resource>] the flattened enumeration of changes
+        def all_changes(of_type: nil, in_range: nil)
+          @change_lists ||= {}
+          lists = in_range ? change_lists(in_range: in_range, strict: false) : resources
+          lists.flat_map do |cl|
+            @change_lists[cl] ||= cl.get_and_parse
+            @change_lists[cl].changes(of_type: of_type, in_range: in_range)
+          end
+        end
+      end
+    end
+  end
+
+  class BaseChangeIndex
+    prepend Client::Mixins::ChangeIndex
+  end
+end

data/lib/resync/client/mixins/zipped_resource.rb

@@ -12,7 +12,6 @@ module Resync
         # it to a temporary file if necessary.
         # @return [Resync::Client::Zip::ZipPackage] the zipped contents of this resource
         def zip_package
-          puts self
           @zip_package ||= Resync::Client::Zip::ZipPackage.new(download_to_temp_file)
         end
       end

data/lib/resync/client/version.rb

@@ -1,6 +1,6 @@
 module Resync
   class Client
     # The version of this gem.
-    VERSION = '0.2.4'
+    VERSION = '0.2.5'
   end
 end

data/spec/data/examples/change-list-1.xml

@@ -0,0 +1,25 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9"
+        xmlns:rs="http://www.openarchives.org/rs/terms/">
+  <rs:ln rel="up"
+         href="http://example.com/dataset1/capabilitylist.xml"/>
+  <rs:md capability="changelist"
+         from="2013-01-01T00:00:00Z"
+         until="2013-01-02T00:00:00Z"/>
+  <url>
+    <loc>http://example.com/res1</loc>
+    <lastmod>2013-01-01T01:00:00Z</lastmod>
+    <rs:md change="updated"
+           hash="md5:51b99b46c9980623b36b27e06a7e4a46"
+           length="446"
+           type="text/plain"/>
+  </url>
+  <url>
+    <loc>http://example.com/res2</loc>
+    <lastmod>2013-01-01T23:00:00Z</lastmod>
+    <rs:md change="updated"
+           hash="md5:6205d276e0c120eb4bac47c14a8297d8"
+           length="447"
+           type="text/plain"/>
+  </url>
+</urlset>

data/spec/data/examples/change-list-2.xml

@@ -0,0 +1,25 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9"
+        xmlns:rs="http://www.openarchives.org/rs/terms/">
+  <rs:ln rel="up"
+         href="http://example.com/dataset1/capabilitylist.xml"/>
+  <rs:md capability="changelist"
+         from="2013-01-02T00:00:00Z"
+         until="2013-01-03T00:00:00Z"/>
+  <url>
+    <loc>http://example.com/res1</loc>
+    <lastmod>2013-01-02T01:00:00Z</lastmod>
+    <rs:md change="updated"
+           hash="md5:6205d276e0c120eb4bac47c14a8297d8"
+           length="447"
+           type="text/plain"/>
+  </url>
+  <url>
+    <loc>http://example.com/res2</loc>
+    <lastmod>2013-01-02T23:00:00Z</lastmod>
+    <rs:md change="updated"
+           hash="md5:51b99b46c9980623b36b27e06a7e4a46"
+           length="446"
+           type="text/plain"/>
+  </url>
+</urlset>

data/spec/data/examples/change-list-3.xml

@@ -0,0 +1,16 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9"
+        xmlns:rs="http://www.openarchives.org/rs/terms/">
+  <rs:ln rel="up"
+         href="http://example.com/dataset1/capabilitylist.xml"/>
+  <rs:md capability="changelist"
+         from="2013-01-03T00:00:00Z"/>
+  <url>
+    <loc>http://example.com/res1</loc>
+    <lastmod>2013-01-03T01:00:00Z</lastmod>
+    <rs:md change="updated"
+           hash="md5:51b99b46c9980623b36b27e06a7e4a46"
+           length="446"
+           type="text/plain"/>
+  </url>
+</urlset>

data/spec/rspec_custom_matchers.rb

@@ -9,3 +9,25 @@ RSpec::Matchers.define :request_for do |h|
     end
   end
 end
+
+RSpec::Matchers.define :be_time do |expected|
+
+  def to_string(time)
+    time.is_a?(Time) ? time.utc.round(2).iso8601(2) : time.to_s
+  end
+
+  match do |actual|
+    if expected
+      fail "Expected value #{expected} is not a Time" unless expected.is_a?(Time)
+      actual.is_a?(Time) && (to_string(expected) == to_string(actual))
+    else
+      return actual.nil?
+    end
+  end
+
+  failure_message do |actual|
+    expected_str = to_string(expected)
+    actual_str = to_string(actual)
+    "expected time:\n#{expected_str}\n\nbut was:\n#{actual_str}"
+  end
+end

data/spec/unit/resync/client/extensions_spec.rb

@@ -41,6 +41,74 @@ module Resync
       end
     end
 
+    describe BaseChangeIndex do
+      before(:each) do
+        @helper = instance_double(Client::HTTPHelper)
+        @client = Client.new(helper: @helper)
+      end
+
+      describe '#all_changes' do
+        it 'flattens the child changelists' do
+          change_index_uri = URI('http://example.com/dataset1/changelist.xml')
+          change_index_data = File.read('spec/data/examples/change-list-index.xml')
+          expect(@helper).to receive(:fetch).with(uri: change_index_uri).and_return(change_index_data)
+
+          list1_uri = URI('http://example.com/20130101-changelist.xml')
+          list1_data = File.read('spec/data/examples/change-list-1.xml')
+          expect(@helper).to receive(:fetch).with(uri: list1_uri).and_return(list1_data)
+
+          list2_uri = URI('http://example.com/20130102-changelist.xml')
+          list2_data = File.read('spec/data/examples/change-list-2.xml')
+          expect(@helper).to receive(:fetch).with(uri: list2_uri).and_return(list2_data)
+
+          list3_uri = URI('http://example.com/20130103-changelist.xml')
+          list3_data = File.read('spec/data/examples/change-list-3.xml')
+          expect(@helper).to receive(:fetch).with(uri: list3_uri).and_return(list3_data)
+
+          expected_mtimes = [
+            Time.utc(2013, 1, 1, 1),
+            Time.utc(2013, 1, 1, 23),
+            Time.utc(2013, 1, 2, 1),
+            Time.utc(2013, 1, 2, 23),
+            Time.utc(2013, 1, 3, 1)
+          ]
+
+          change_index = @client.get_and_parse(change_index_uri)
+          index = 0
+          change_index.all_changes(in_range: (Time.utc(0)..Time.new)).each do |c|
+            res = index.even? ? 1 : 2
+            expect(c.uri).to eq(URI("http://example.com/res#{res}"))
+            expect(c.modified_time).to be_time(expected_mtimes[index])
+            index += 1
+          end
+          expect(index).to eq(5)
+        end
+
+        it 'doesn\'t download unnecessary changelists' do
+          change_index_uri = URI('http://example.com/dataset1/changelist.xml')
+          change_index_data = File.read('spec/data/examples/change-list-index.xml')
+          expect(@helper).to receive(:fetch).with(uri: change_index_uri).and_return(change_index_data)
+
+          list2_uri = URI('http://example.com/20130102-changelist.xml')
+          list2_data = File.read('spec/data/examples/change-list-2.xml')
+          expect(@helper).to receive(:fetch).with(uri: list2_uri).and_return(list2_data)
+
+          list3_uri = URI('http://example.com/20130103-changelist.xml')
+          list3_data = File.read('spec/data/examples/change-list-3.xml')
+          expect(@helper).to receive(:fetch).with(uri: list3_uri).and_return(list3_data)
+
+          change_index = @client.get_and_parse(change_index_uri)
+          count = 0
+          change_index.all_changes(in_range: (Time.utc(2013, 1, 2, 12)..Time.utc(2013, 1, 3, 0, 30))).each do |c|
+            expect(c.modified_time).to be_time(Time.utc(2013, 1, 2, 23))
+            expect(c.uri).to eq(URI('http://example.com/res2'))
+            count += 1
+          end
+          expect(count).to eq(1)
+        end
+      end
+    end
+
     describe Augmented do
       it 'proxies all link client requests to its own client' do
         client = instance_double(Resync::Client)

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: resync-client
 version: !ruby/object:Gem::Version
-  version: 0.2.4
+  version: 0.2.5
 platform: ruby
 authors:
 - David Moles
@@ -163,6 +163,7 @@ files:
 - lib/resync/client/http_helper.rb
 - lib/resync/client/mixins.rb
 - lib/resync/client/mixins/bitstream_resource.rb
+- lib/resync/client/mixins/change_index.rb
 - lib/resync/client/mixins/client_delegator.rb
 - lib/resync/client/mixins/downloadable.rb
 - lib/resync/client/mixins/dump.rb
@@ -182,6 +183,9 @@ files:
 - spec/data/examples/capability-list.xml
 - spec/data/examples/change-dump-manifest.xml
 - spec/data/examples/change-dump.xml
+- spec/data/examples/change-list-1.xml
+- spec/data/examples/change-list-2.xml
+- spec/data/examples/change-list-3.xml
 - spec/data/examples/change-list-index.xml
 - spec/data/examples/change-list.xml
 - spec/data/examples/resource-dump-manifest.xml
@@ -243,6 +247,9 @@ test_files:
 - spec/data/examples/capability-list.xml
 - spec/data/examples/change-dump-manifest.xml
 - spec/data/examples/change-dump.xml
+- spec/data/examples/change-list-1.xml
+- spec/data/examples/change-list-2.xml
+- spec/data/examples/change-list-3.xml
 - spec/data/examples/change-list-index.xml
 - spec/data/examples/change-list.xml
 - spec/data/examples/resource-dump-manifest.xml