pho 0.7.1 → 0.7.2

data/lib/pho.rb

@@ -2,6 +2,7 @@ require 'rubygems'
 require 'httpclient'
 require 'json'
 require 'yaml'
+require 'date'
 require 'rexml/document'
 
 require 'pho/etags'

data/lib/pho/oai.rb

@@ -6,11 +6,11 @@ module Pho
     class Record
       
         attr_reader :identifier
-        attr_reader :timestamp
+        attr_reader :datestamp
         
-        def initialize(identifier, timestamp)
+        def initialize(identifier, datestamp)
           @identifier = identifier
-          @timestamp = timestamp
+          @datestamp = datestamp
         end  
         
     end
@@ -18,39 +18,76 @@ module Pho
     #Collection of records
     class Records
     
-      attr_reader :responseDate, :from, :to, :records, :token
+      attr_reader :response_date, :from, :to, :records, :resumption_token, :list_size, :cursor
       
-      def initialize(responseDate, token, from, to, records=[])
-        @responseDate = responseDate
-        @token = token
+      def initialize(responseDate, from, to, records=[], token=nil, list_size=nil, cursor=nil)
+        @response_date = responseDate
         @from = from
         @to = to
         @records = records
+        @resumption_token = token
+        @list_size = list_size
+        @cursor = cursor
       end
       
       def Records.parse(response)
         doc = REXML::Document.new(response)
-        records = []
+        if doc.root.get_elements("error")[0] != nil
+          code = doc.root.get_elements("error")[0].attributes["code"]
+          if code == "noRecordsMatch"
+            return nil
+          else
+            raise "Unable to list records: #{code}, #{doc.root.get_elements("error")[0].text}"
+          end          
+        end
+        records = []        
+        responseDate = doc.root.get_elements("responseDate")[0].text
+        from = DateTime.parse( doc.root.get_elements("request")[0].attributes["from"] )
+        to = doc.root.get_elements("request")[0].attributes["until"]
+        to = DateTime.parse( to ) unless to == nil 
         REXML::XPath.each( doc.root,  "//oai:header", {"oai" => "http://www.openarchives.org/OAI/2.0/"} ) do |header|
          uri = header.get_elements("identifier")[0].text
          datestamp = header.get_elements("datestamp")[0].text
-         records << Record.new(uri, datestamp)
+         records << Record.new(uri, DateTime.parse( datestamp ) )
+        end
+        resumption_token = doc.root.get_elements("//resumptionToken")[0]
+        if resumption_token != nil
+            token = resumption_token.text
+            list_size = resumption_token.attributes["completeListSize"].to_i if resumption_token.attributes["completeListSize"] != nil 
+            cursor = resumption_token.attributes["cursor"].to_i if resumption_token.attributes["cursor"] != nil
         end
-        #FIXME
-        return Records.new(nil, nil, nil, nil, records)   
+        return Records.new(DateTime.parse(responseDate), from, to, records, token, list_size, cursor)   
       end
       
       #List records from a store
-      #TODO support for from/to dates
-      #TODO support for resumption tokens
-      def Records.read_from_store(store)
-        resp = store.list_records
+      def Records.read_from_store(store, from=nil, to=nil, resumption_token=nil)
+        resp = store.list_records(from, to, resumption_token)
         if resp.status != 200
           raise "Unable to list records"
         end
         return parse(resp.content)
       end
-        
+      
+      #Fetch the next list of records from a store, using a resumption token
+      #
+      #Returns nil if there are no records to retrieve. Can be used as a simple iterator, e.g:
+      #
+      # records = store.list_records
+      # while records != nil
+      #   #do something with retrieved records
+      #   records = Records.read_next_records(store, records.resumption_token 
+      # end
+      #
+      #store:: the store
+      #records:: previously retrieved resumption_token
+      def Records.read_next_records(store, resumption_token)
+        #list is already complete
+        if records.resumption_token = nil
+          return nil
+        end
+        return Records.read_from_store(store, nil, nil, token)
+      end
+            
     end
     
   end  

data/lib/pho/store.rb

@@ -443,9 +443,19 @@ module Pho
     # OAI
     #############
     
-    def list_records
+    def list_records(from=nil, to=nil, resumption_token=nil)      
       u = build_uri("/services/oai-pmh")
-      response = @client.get(u, {"verb" => "ListRecords", "metadataPrefix" => "oai_dc"})
+      params = {"verb" => "ListRecords", "metadataPrefix" => "oai_dc"}
+      if from != nil
+        params["from"] = from.strftime("%Y-%m-%dT%H:%M:%SZ") if from.respond_to? :strftime
+        params["from"] = from.to_s if !from.respond_to? :strftime
+      end
+      if to != nil
+        params["until"] = to.strftime("%Y-%m-%dT%H:%M:%SZ") if to.respond_to? :strftime
+        params["until"] = to.to_s if !to.respond_to? :strftime
+      end
+      params["resumptionToken"] = resumption_token if resumption_token != nil
+      response = @client.get(u, params)
       return response
     end
         

data/tests/tc_oai.rb

@@ -3,13 +3,50 @@ require 'pho'
 require 'test/unit'
 require 'mocha'
 require 'rexml/document'
-
+require 'date'
 class OAITest < Test::Unit::TestCase
 
+NO_RECORDS = <<-EOL
+  <OAI-PMH xmlns="http://www.openarchives.org/OAI/2.0/" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" 
+   xsi:schemaLocation="http://www.openarchives.org/OAI/2.0/ http://www.openarchives.org/OAI/2.0/OAI-PMH.xsd">
+   <responseDate>2010-04-04T14:33:00Z</responseDate>
+   <request from="2010-04-04T14:33:00Z" metadataPrefix="oai_dc" verb="ListRecords">http://api.talis.com/stores/ldodds-dev2/services/oai-pmh</request>
+   <error code="noRecordsMatch">No matching records were found</error></OAI-PMH>
+EOL
+
 LIST_RECORDS = <<-EOL 
-  <OAI-PMH xmlns="http://www.openarchives.org/OAI/2.0/" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://www.openarchives.org/OAI/2.0/ http://www.openarchives.org/OAI/2.0/OAI-PMH.xsd"><responseDate>2010-03-19T13:02:14Z</responseDate><request from="2010-03-19T13:02:14Z" metadataPrefix="oai_dc" verb="ListRecords">http://api.talis.com/stores/ldodds-dev1/services/oai-pmh</request><ListRecords><record><header><identifier>http://www.example.org</identifier><datestamp>2010-03-19T12:57:06Z</datestamp></header><metadata><oai_dc:dc xmlns:dc="http://purl.org/dc/elements/1.1/" xmlns:oai_dc="http://www.openarchives.org/OAI/2.0/oai_dc/"><dc:identifier>http://www.example.org</dc:identifier></oai_dc:dc></metadata></record><record><header><identifier>http://api.talis.com/stores/ldodds-dev1/items/atom.xml</identifier><datestamp>2010-03-19T12:57:10Z</datestamp></header><metadata><oai_dc:dc xmlns:dc="http://purl.org/dc/elements/1.1/" xmlns:oai_dc="http://www.openarchives.org/OAI/2.0/oai_dc/"><dc:identifier>http://api.talis.com/stores/ldodds-dev1/items/atom.xml</dc:identifier></oai_dc:dc></metadata></record></ListRecords></OAI-PMH> 
+  <OAI-PMH xmlns="http://www.openarchives.org/OAI/2.0/" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://www.openarchives.org/OAI/2.0/ http://www.openarchives.org/OAI/2.0/OAI-PMH.xsd">
+  <responseDate>2010-03-19T13:02:14Z</responseDate>
+  <request from="2010-03-19T13:02:14Z" metadataPrefix="oai_dc" verb="ListRecords">http://api.talis.com/stores/ldodds-dev1/services/oai-pmh</request>
+  <ListRecords>
+  <record><header><identifier>http://www.example.org</identifier><datestamp>2010-03-19T12:57:06Z</datestamp></header>
+  <metadata><oai_dc:dc xmlns:dc="http://purl.org/dc/elements/1.1/" xmlns:oai_dc="http://www.openarchives.org/OAI/2.0/oai_dc/">
+  <dc:identifier>http://www.example.org</dc:identifier></oai_dc:dc></metadata>
+  </record>
+  <record><header><identifier>http://api.talis.com/stores/ldodds-dev1/items/atom.xml</identifier><datestamp>2010-03-19T12:57:10Z</datestamp></header>
+  <metadata><oai_dc:dc xmlns:dc="http://purl.org/dc/elements/1.1/" xmlns:oai_dc="http://www.openarchives.org/OAI/2.0/oai_dc/">
+  <dc:identifier>http://api.talis.com/stores/ldodds-dev1/items/atom.xml</dc:identifier></oai_dc:dc></metadata>
+  </record>  
+  </ListRecords></OAI-PMH> 
 EOL
 
+LIST_RECORDS_WITH_TOKEN = <<-EOL 
+    <OAI-PMH xmlns="http://www.openarchives.org/OAI/2.0/" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://www.openarchives.org/OAI/2.0/ http://www.openarchives.org/OAI/2.0/OAI-PMH.xsd">
+    <responseDate>2010-03-19T13:02:14Z</responseDate>
+    <request from="2010-03-19T13:02:14Z" metadataPrefix="oai_dc" verb="ListRecords">http://api.talis.com/stores/ldodds-dev1/services/oai-pmh</request>
+    <ListRecords>
+    <record><header><identifier>http://www.example.org</identifier><datestamp>2010-03-19T12:57:06Z</datestamp></header>
+    <metadata><oai_dc:dc xmlns:dc="http://purl.org/dc/elements/1.1/" xmlns:oai_dc="http://www.openarchives.org/OAI/2.0/oai_dc/">
+    <dc:identifier>http://www.example.org</dc:identifier></oai_dc:dc></metadata>
+    </record>
+    <record><header><identifier>http://api.talis.com/stores/ldodds-dev1/items/atom.xml</identifier><datestamp>2010-03-19T12:57:10Z</datestamp></header>
+    <metadata><oai_dc:dc xmlns:dc="http://purl.org/dc/elements/1.1/" xmlns:oai_dc="http://www.openarchives.org/OAI/2.0/oai_dc/">
+    <dc:identifier>http://api.talis.com/stores/ldodds-dev1/items/atom.xml</dc:identifier></oai_dc:dc></metadata>
+    </record>
+    <resumptionToken completeListSize="6151294" cursor="0">b2FpX2RjfDEwMHwxOTcwLTAxLTAxVDAwOjAwOjAwWnwyMDEwLTA0LTA0VDEzOjU0OjMyWg==</resumptionToken>  
+    </ListRecords></OAI-PMH> 
+EOL
+  
   def test_request
     mc = mock()
     mc.expects(:set_auth)
@@ -21,11 +58,81 @@ EOL
     assert_equal(LIST_RECORDS, response.content)         
   end    
   
+  def test_parse_no_records
+    results = Pho::OAI::Records.parse(NO_RECORDS)
+    assert_nil(results)    
+  end
+  
+  def test_request_with_from
+    mc = mock()
+    mc.expects(:set_auth)
+    mc.expects(:get).with("http://api.talis.com/stores/testing/services/oai-pmh", {"verb" => "ListRecords", "metadataPrefix" => "oai_dc", "from" => "2010-03-19T13:02:14Z"} ).returns(
+      HTTP::Message.new_response(LIST_RECORDS))
+    
+    store = Pho::Store.new("http://api.talis.com/stores/testing", "user", "pass", mc)
+    response = store.list_records("2010-03-19T13:02:14Z")
+    assert_equal(LIST_RECORDS, response.content)         
+
+    mc = mock()
+    mc.expects(:set_auth)
+    mc.expects(:get).with("http://api.talis.com/stores/testing/services/oai-pmh", {"verb" => "ListRecords", "metadataPrefix" => "oai_dc", "from" => "2010-03-19T13:02:14Z"} ).returns(
+      HTTP::Message.new_response(LIST_RECORDS))
+        
+    store = Pho::Store.new("http://api.talis.com/stores/testing", "user", "pass", mc)
+    response = store.list_records(DateTime.parse("2010-03-19T13:02:14Z"))
+    assert_equal(LIST_RECORDS, response.content)       
+  end 
+
+  def test_request_with_to
+    mc = mock()
+    mc.expects(:set_auth)
+    mc.expects(:get).with("http://api.talis.com/stores/testing/services/oai-pmh", {"verb" => "ListRecords", "metadataPrefix" => "oai_dc", "until" => "2010-03-19T13:02:14Z"} ).returns(
+      HTTP::Message.new_response(LIST_RECORDS))
+    
+    store = Pho::Store.new("http://api.talis.com/stores/testing", "user", "pass", mc)
+    response = store.list_records(nil, "2010-03-19T13:02:14Z")
+    assert_equal(LIST_RECORDS, response.content)         
+
+    mc = mock()
+    mc.expects(:set_auth)
+    mc.expects(:get).with("http://api.talis.com/stores/testing/services/oai-pmh", {"verb" => "ListRecords", "metadataPrefix" => "oai_dc", "until" => "2010-03-19T13:02:14Z"} ).returns(
+      HTTP::Message.new_response(LIST_RECORDS))
+    store = Pho::Store.new("http://api.talis.com/stores/testing", "user", "pass", mc)
+    response = store.list_records(nil, DateTime.parse("2010-03-19T13:02:14Z"))  
+    assert_equal(LIST_RECORDS, response.content)               
+  end
+        
+  def test_request_with_token
+    mc = mock()
+    mc.expects(:set_auth)
+    mc.expects(:get).with("http://api.talis.com/stores/testing/services/oai-pmh", {"verb" => "ListRecords", "metadataPrefix" => "oai_dc", "resumptionToken" => "abc"} ).returns(
+      HTTP::Message.new_response(LIST_RECORDS))
+    
+    store = Pho::Store.new("http://api.talis.com/stores/testing", "user", "pass", mc)
+    response = store.list_records(nil, nil, "abc")
+    assert_equal(LIST_RECORDS, response.content)         
+  end
+    
   def test_parse
   
     results = Pho::OAI::Records.parse(LIST_RECORDS)
     assert_not_nil(results)
+    assert_equal(2010, results.from.year)
+    assert_equal(3, results.from.month)
+    assert_equal(nil, results.to)
     assert_equal(2, results.records.size)  
+    assert_equal("http://www.example.org", results.records[0].identifier)    
+    assert_equal(2010, results.records[0].datestamp.year)
+    assert_equal("http://api.talis.com/stores/ldodds-dev1/items/atom.xml", results.records[1].identifier)
+    assert_equal(2010, results.records[1].datestamp.year)
   end
   
+  def test_parse_with_token
+  
+    results = Pho::OAI::Records.parse(LIST_RECORDS_WITH_TOKEN)
+    assert_not_nil(results)
+    assert_equal(6151294, results.list_size)
+    assert_equal(0, results.cursor)
+    assert_equal("b2FpX2RjfDEwMHwxOTcwLTAxLTAxVDAwOjAwOjAwWnwyMDEwLTA0LTA0VDEzOjU0OjMyWg==", results.resumption_token)
+  end  
 end

metadata

@@ -5,8 +5,8 @@ version: !ruby/object:Gem::Version
   segments: 
   - 0
   - 7
-  - 1
-  version: 0.7.1
+  - 2
+  version: 0.7.2
 platform: ruby
 authors: 
 - Leigh Dodds
@@ -14,7 +14,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2010-04-03 00:00:00 +01:00
+date: 2010-04-04 00:00:00 +01:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency