pho 0.7.1 → 0.7.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (35) hide show
  1. data/CHANGES +4 -0
  2. data/Rakefile +1 -1
  3. data/bin/talis_store +1 -1
  4. data/doc/rdoc/classes/Pho/CommandLine.html +108 -108
  5. data/doc/rdoc/classes/Pho/DatatypeProperty.html +12 -12
  6. data/doc/rdoc/classes/Pho/Enrichment/StoreEnricher.html +29 -29
  7. data/doc/rdoc/classes/Pho/Etags.html +36 -36
  8. data/doc/rdoc/classes/Pho/Facet/Results.html +19 -19
  9. data/doc/rdoc/classes/Pho/Facet/Term.html +6 -6
  10. data/doc/rdoc/classes/Pho/FieldPredicateMap.html +90 -90
  11. data/doc/rdoc/classes/Pho/FieldWeighting.html +12 -12
  12. data/doc/rdoc/classes/Pho/Job.html +64 -64
  13. data/doc/rdoc/classes/Pho/Jobs.html +60 -60
  14. data/doc/rdoc/classes/Pho/OAI/Record.html +11 -11
  15. data/doc/rdoc/classes/Pho/OAI/Records.html +96 -18
  16. data/doc/rdoc/classes/Pho/QueryProfile.html +60 -60
  17. data/doc/rdoc/classes/Pho/RDF/Parser.html +24 -24
  18. data/doc/rdoc/classes/Pho/ResourceHash/Converter.html +36 -36
  19. data/doc/rdoc/classes/Pho/ResourceHash/SetAlgebra.html +12 -12
  20. data/doc/rdoc/classes/Pho/Snapshot.html +35 -35
  21. data/doc/rdoc/classes/Pho/Status.html +26 -26
  22. data/doc/rdoc/classes/Pho/Store.html +232 -222
  23. data/doc/rdoc/classes/Pho/StoreSparqlClient.html +14 -14
  24. data/doc/rdoc/classes/String.html +1 -1
  25. data/doc/rdoc/created.rid +1 -1
  26. data/doc/rdoc/files/CHANGES.html +8 -1
  27. data/doc/rdoc/files/lib/pho/oai_rb.html +1 -1
  28. data/doc/rdoc/files/lib/pho/store_rb.html +1 -1
  29. data/doc/rdoc/files/lib/pho_rb.html +2 -1
  30. data/doc/rdoc/fr_method_index.html +148 -147
  31. data/lib/pho.rb +1 -0
  32. data/lib/pho/oai.rb +53 -16
  33. data/lib/pho/store.rb +12 -2
  34. data/tests/tc_oai.rb +109 -2
  35. metadata +3 -3
data/lib/pho.rb CHANGED
@@ -2,6 +2,7 @@ require 'rubygems'
2
2
  require 'httpclient'
3
3
  require 'json'
4
4
  require 'yaml'
5
+ require 'date'
5
6
  require 'rexml/document'
6
7
 
7
8
  require 'pho/etags'
@@ -6,11 +6,11 @@ module Pho
6
6
  class Record
7
7
 
8
8
  attr_reader :identifier
9
- attr_reader :timestamp
9
+ attr_reader :datestamp
10
10
 
11
- def initialize(identifier, timestamp)
11
+ def initialize(identifier, datestamp)
12
12
  @identifier = identifier
13
- @timestamp = timestamp
13
+ @datestamp = datestamp
14
14
  end
15
15
 
16
16
  end
@@ -18,39 +18,76 @@ module Pho
18
18
  #Collection of records
19
19
  class Records
20
20
 
21
- attr_reader :responseDate, :from, :to, :records, :token
21
+ attr_reader :response_date, :from, :to, :records, :resumption_token, :list_size, :cursor
22
22
 
23
- def initialize(responseDate, token, from, to, records=[])
24
- @responseDate = responseDate
25
- @token = token
23
+ def initialize(responseDate, from, to, records=[], token=nil, list_size=nil, cursor=nil)
24
+ @response_date = responseDate
26
25
  @from = from
27
26
  @to = to
28
27
  @records = records
28
+ @resumption_token = token
29
+ @list_size = list_size
30
+ @cursor = cursor
29
31
  end
30
32
 
31
33
  def Records.parse(response)
32
34
  doc = REXML::Document.new(response)
33
- records = []
35
+ if doc.root.get_elements("error")[0] != nil
36
+ code = doc.root.get_elements("error")[0].attributes["code"]
37
+ if code == "noRecordsMatch"
38
+ return nil
39
+ else
40
+ raise "Unable to list records: #{code}, #{doc.root.get_elements("error")[0].text}"
41
+ end
42
+ end
43
+ records = []
44
+ responseDate = doc.root.get_elements("responseDate")[0].text
45
+ from = DateTime.parse( doc.root.get_elements("request")[0].attributes["from"] )
46
+ to = doc.root.get_elements("request")[0].attributes["until"]
47
+ to = DateTime.parse( to ) unless to == nil
34
48
  REXML::XPath.each( doc.root, "//oai:header", {"oai" => "http://www.openarchives.org/OAI/2.0/"} ) do |header|
35
49
  uri = header.get_elements("identifier")[0].text
36
50
  datestamp = header.get_elements("datestamp")[0].text
37
- records << Record.new(uri, datestamp)
51
+ records << Record.new(uri, DateTime.parse( datestamp ) )
52
+ end
53
+ resumption_token = doc.root.get_elements("//resumptionToken")[0]
54
+ if resumption_token != nil
55
+ token = resumption_token.text
56
+ list_size = resumption_token.attributes["completeListSize"].to_i if resumption_token.attributes["completeListSize"] != nil
57
+ cursor = resumption_token.attributes["cursor"].to_i if resumption_token.attributes["cursor"] != nil
38
58
  end
39
- #FIXME
40
- return Records.new(nil, nil, nil, nil, records)
59
+ return Records.new(DateTime.parse(responseDate), from, to, records, token, list_size, cursor)
41
60
  end
42
61
 
43
62
  #List records from a store
44
- #TODO support for from/to dates
45
- #TODO support for resumption tokens
46
- def Records.read_from_store(store)
47
- resp = store.list_records
63
+ def Records.read_from_store(store, from=nil, to=nil, resumption_token=nil)
64
+ resp = store.list_records(from, to, resumption_token)
48
65
  if resp.status != 200
49
66
  raise "Unable to list records"
50
67
  end
51
68
  return parse(resp.content)
52
69
  end
53
-
70
+
71
+ #Fetch the next list of records from a store, using a resumption token
72
+ #
73
+ #Returns nil if there are no records to retrieve. Can be used as a simple iterator, e.g:
74
+ #
75
+ # records = store.list_records
76
+ # while records != nil
77
+ # #do something with retrieved records
78
+ # records = Records.read_next_records(store, records.resumption_token
79
+ # end
80
+ #
81
+ #store:: the store
82
+ #records:: previously retrieved resumption_token
83
+ def Records.read_next_records(store, resumption_token)
84
+ #list is already complete
85
+ if records.resumption_token = nil
86
+ return nil
87
+ end
88
+ return Records.read_from_store(store, nil, nil, token)
89
+ end
90
+
54
91
  end
55
92
 
56
93
  end
@@ -443,9 +443,19 @@ module Pho
443
443
  # OAI
444
444
  #############
445
445
 
446
- def list_records
446
+ def list_records(from=nil, to=nil, resumption_token=nil)
447
447
  u = build_uri("/services/oai-pmh")
448
- response = @client.get(u, {"verb" => "ListRecords", "metadataPrefix" => "oai_dc"})
448
+ params = {"verb" => "ListRecords", "metadataPrefix" => "oai_dc"}
449
+ if from != nil
450
+ params["from"] = from.strftime("%Y-%m-%dT%H:%M:%SZ") if from.respond_to? :strftime
451
+ params["from"] = from.to_s if !from.respond_to? :strftime
452
+ end
453
+ if to != nil
454
+ params["until"] = to.strftime("%Y-%m-%dT%H:%M:%SZ") if to.respond_to? :strftime
455
+ params["until"] = to.to_s if !to.respond_to? :strftime
456
+ end
457
+ params["resumptionToken"] = resumption_token if resumption_token != nil
458
+ response = @client.get(u, params)
449
459
  return response
450
460
  end
451
461
 
@@ -3,13 +3,50 @@ require 'pho'
3
3
  require 'test/unit'
4
4
  require 'mocha'
5
5
  require 'rexml/document'
6
-
6
+ require 'date'
7
7
  class OAITest < Test::Unit::TestCase
8
8
 
9
+ NO_RECORDS = <<-EOL
10
+ <OAI-PMH xmlns="http://www.openarchives.org/OAI/2.0/" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
11
+ xsi:schemaLocation="http://www.openarchives.org/OAI/2.0/ http://www.openarchives.org/OAI/2.0/OAI-PMH.xsd">
12
+ <responseDate>2010-04-04T14:33:00Z</responseDate>
13
+ <request from="2010-04-04T14:33:00Z" metadataPrefix="oai_dc" verb="ListRecords">http://api.talis.com/stores/ldodds-dev2/services/oai-pmh</request>
14
+ <error code="noRecordsMatch">No matching records were found</error></OAI-PMH>
15
+ EOL
16
+
9
17
  LIST_RECORDS = <<-EOL
10
- <OAI-PMH xmlns="http://www.openarchives.org/OAI/2.0/" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://www.openarchives.org/OAI/2.0/ http://www.openarchives.org/OAI/2.0/OAI-PMH.xsd"><responseDate>2010-03-19T13:02:14Z</responseDate><request from="2010-03-19T13:02:14Z" metadataPrefix="oai_dc" verb="ListRecords">http://api.talis.com/stores/ldodds-dev1/services/oai-pmh</request><ListRecords><record><header><identifier>http://www.example.org</identifier><datestamp>2010-03-19T12:57:06Z</datestamp></header><metadata><oai_dc:dc xmlns:dc="http://purl.org/dc/elements/1.1/" xmlns:oai_dc="http://www.openarchives.org/OAI/2.0/oai_dc/"><dc:identifier>http://www.example.org</dc:identifier></oai_dc:dc></metadata></record><record><header><identifier>http://api.talis.com/stores/ldodds-dev1/items/atom.xml</identifier><datestamp>2010-03-19T12:57:10Z</datestamp></header><metadata><oai_dc:dc xmlns:dc="http://purl.org/dc/elements/1.1/" xmlns:oai_dc="http://www.openarchives.org/OAI/2.0/oai_dc/"><dc:identifier>http://api.talis.com/stores/ldodds-dev1/items/atom.xml</dc:identifier></oai_dc:dc></metadata></record></ListRecords></OAI-PMH>
18
+ <OAI-PMH xmlns="http://www.openarchives.org/OAI/2.0/" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://www.openarchives.org/OAI/2.0/ http://www.openarchives.org/OAI/2.0/OAI-PMH.xsd">
19
+ <responseDate>2010-03-19T13:02:14Z</responseDate>
20
+ <request from="2010-03-19T13:02:14Z" metadataPrefix="oai_dc" verb="ListRecords">http://api.talis.com/stores/ldodds-dev1/services/oai-pmh</request>
21
+ <ListRecords>
22
+ <record><header><identifier>http://www.example.org</identifier><datestamp>2010-03-19T12:57:06Z</datestamp></header>
23
+ <metadata><oai_dc:dc xmlns:dc="http://purl.org/dc/elements/1.1/" xmlns:oai_dc="http://www.openarchives.org/OAI/2.0/oai_dc/">
24
+ <dc:identifier>http://www.example.org</dc:identifier></oai_dc:dc></metadata>
25
+ </record>
26
+ <record><header><identifier>http://api.talis.com/stores/ldodds-dev1/items/atom.xml</identifier><datestamp>2010-03-19T12:57:10Z</datestamp></header>
27
+ <metadata><oai_dc:dc xmlns:dc="http://purl.org/dc/elements/1.1/" xmlns:oai_dc="http://www.openarchives.org/OAI/2.0/oai_dc/">
28
+ <dc:identifier>http://api.talis.com/stores/ldodds-dev1/items/atom.xml</dc:identifier></oai_dc:dc></metadata>
29
+ </record>
30
+ </ListRecords></OAI-PMH>
11
31
  EOL
12
32
 
33
+ LIST_RECORDS_WITH_TOKEN = <<-EOL
34
+ <OAI-PMH xmlns="http://www.openarchives.org/OAI/2.0/" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://www.openarchives.org/OAI/2.0/ http://www.openarchives.org/OAI/2.0/OAI-PMH.xsd">
35
+ <responseDate>2010-03-19T13:02:14Z</responseDate>
36
+ <request from="2010-03-19T13:02:14Z" metadataPrefix="oai_dc" verb="ListRecords">http://api.talis.com/stores/ldodds-dev1/services/oai-pmh</request>
37
+ <ListRecords>
38
+ <record><header><identifier>http://www.example.org</identifier><datestamp>2010-03-19T12:57:06Z</datestamp></header>
39
+ <metadata><oai_dc:dc xmlns:dc="http://purl.org/dc/elements/1.1/" xmlns:oai_dc="http://www.openarchives.org/OAI/2.0/oai_dc/">
40
+ <dc:identifier>http://www.example.org</dc:identifier></oai_dc:dc></metadata>
41
+ </record>
42
+ <record><header><identifier>http://api.talis.com/stores/ldodds-dev1/items/atom.xml</identifier><datestamp>2010-03-19T12:57:10Z</datestamp></header>
43
+ <metadata><oai_dc:dc xmlns:dc="http://purl.org/dc/elements/1.1/" xmlns:oai_dc="http://www.openarchives.org/OAI/2.0/oai_dc/">
44
+ <dc:identifier>http://api.talis.com/stores/ldodds-dev1/items/atom.xml</dc:identifier></oai_dc:dc></metadata>
45
+ </record>
46
+ <resumptionToken completeListSize="6151294" cursor="0">b2FpX2RjfDEwMHwxOTcwLTAxLTAxVDAwOjAwOjAwWnwyMDEwLTA0LTA0VDEzOjU0OjMyWg==</resumptionToken>
47
+ </ListRecords></OAI-PMH>
48
+ EOL
49
+
13
50
  def test_request
14
51
  mc = mock()
15
52
  mc.expects(:set_auth)
@@ -21,11 +58,81 @@ EOL
21
58
  assert_equal(LIST_RECORDS, response.content)
22
59
  end
23
60
 
61
+ def test_parse_no_records
62
+ results = Pho::OAI::Records.parse(NO_RECORDS)
63
+ assert_nil(results)
64
+ end
65
+
66
+ def test_request_with_from
67
+ mc = mock()
68
+ mc.expects(:set_auth)
69
+ mc.expects(:get).with("http://api.talis.com/stores/testing/services/oai-pmh", {"verb" => "ListRecords", "metadataPrefix" => "oai_dc", "from" => "2010-03-19T13:02:14Z"} ).returns(
70
+ HTTP::Message.new_response(LIST_RECORDS))
71
+
72
+ store = Pho::Store.new("http://api.talis.com/stores/testing", "user", "pass", mc)
73
+ response = store.list_records("2010-03-19T13:02:14Z")
74
+ assert_equal(LIST_RECORDS, response.content)
75
+
76
+ mc = mock()
77
+ mc.expects(:set_auth)
78
+ mc.expects(:get).with("http://api.talis.com/stores/testing/services/oai-pmh", {"verb" => "ListRecords", "metadataPrefix" => "oai_dc", "from" => "2010-03-19T13:02:14Z"} ).returns(
79
+ HTTP::Message.new_response(LIST_RECORDS))
80
+
81
+ store = Pho::Store.new("http://api.talis.com/stores/testing", "user", "pass", mc)
82
+ response = store.list_records(DateTime.parse("2010-03-19T13:02:14Z"))
83
+ assert_equal(LIST_RECORDS, response.content)
84
+ end
85
+
86
+ def test_request_with_to
87
+ mc = mock()
88
+ mc.expects(:set_auth)
89
+ mc.expects(:get).with("http://api.talis.com/stores/testing/services/oai-pmh", {"verb" => "ListRecords", "metadataPrefix" => "oai_dc", "until" => "2010-03-19T13:02:14Z"} ).returns(
90
+ HTTP::Message.new_response(LIST_RECORDS))
91
+
92
+ store = Pho::Store.new("http://api.talis.com/stores/testing", "user", "pass", mc)
93
+ response = store.list_records(nil, "2010-03-19T13:02:14Z")
94
+ assert_equal(LIST_RECORDS, response.content)
95
+
96
+ mc = mock()
97
+ mc.expects(:set_auth)
98
+ mc.expects(:get).with("http://api.talis.com/stores/testing/services/oai-pmh", {"verb" => "ListRecords", "metadataPrefix" => "oai_dc", "until" => "2010-03-19T13:02:14Z"} ).returns(
99
+ HTTP::Message.new_response(LIST_RECORDS))
100
+ store = Pho::Store.new("http://api.talis.com/stores/testing", "user", "pass", mc)
101
+ response = store.list_records(nil, DateTime.parse("2010-03-19T13:02:14Z"))
102
+ assert_equal(LIST_RECORDS, response.content)
103
+ end
104
+
105
+ def test_request_with_token
106
+ mc = mock()
107
+ mc.expects(:set_auth)
108
+ mc.expects(:get).with("http://api.talis.com/stores/testing/services/oai-pmh", {"verb" => "ListRecords", "metadataPrefix" => "oai_dc", "resumptionToken" => "abc"} ).returns(
109
+ HTTP::Message.new_response(LIST_RECORDS))
110
+
111
+ store = Pho::Store.new("http://api.talis.com/stores/testing", "user", "pass", mc)
112
+ response = store.list_records(nil, nil, "abc")
113
+ assert_equal(LIST_RECORDS, response.content)
114
+ end
115
+
24
116
  def test_parse
25
117
 
26
118
  results = Pho::OAI::Records.parse(LIST_RECORDS)
27
119
  assert_not_nil(results)
120
+ assert_equal(2010, results.from.year)
121
+ assert_equal(3, results.from.month)
122
+ assert_equal(nil, results.to)
28
123
  assert_equal(2, results.records.size)
124
+ assert_equal("http://www.example.org", results.records[0].identifier)
125
+ assert_equal(2010, results.records[0].datestamp.year)
126
+ assert_equal("http://api.talis.com/stores/ldodds-dev1/items/atom.xml", results.records[1].identifier)
127
+ assert_equal(2010, results.records[1].datestamp.year)
29
128
  end
30
129
 
130
+ def test_parse_with_token
131
+
132
+ results = Pho::OAI::Records.parse(LIST_RECORDS_WITH_TOKEN)
133
+ assert_not_nil(results)
134
+ assert_equal(6151294, results.list_size)
135
+ assert_equal(0, results.cursor)
136
+ assert_equal("b2FpX2RjfDEwMHwxOTcwLTAxLTAxVDAwOjAwOjAwWnwyMDEwLTA0LTA0VDEzOjU0OjMyWg==", results.resumption_token)
137
+ end
31
138
  end
metadata CHANGED
@@ -5,8 +5,8 @@ version: !ruby/object:Gem::Version
5
5
  segments:
6
6
  - 0
7
7
  - 7
8
- - 1
9
- version: 0.7.1
8
+ - 2
9
+ version: 0.7.2
10
10
  platform: ruby
11
11
  authors:
12
12
  - Leigh Dodds
@@ -14,7 +14,7 @@ autorequire:
14
14
  bindir: bin
15
15
  cert_chain: []
16
16
 
17
- date: 2010-04-03 00:00:00 +01:00
17
+ date: 2010-04-04 00:00:00 +01:00
18
18
  default_executable:
19
19
  dependencies:
20
20
  - !ruby/object:Gem::Dependency