oai 0.2.1 → 0.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (48) hide show
  1. data/README.md +28 -23
  2. data/Rakefile +14 -40
  3. data/examples/providers/dublin_core.rb +63 -63
  4. data/lib/oai/client.rb +131 -97
  5. data/lib/oai/client/list_identifiers.rb +1 -0
  6. data/lib/oai/client/list_records.rb +6 -5
  7. data/lib/oai/client/list_sets.rb +6 -5
  8. data/lib/oai/client/record.rb +6 -7
  9. data/lib/oai/client/response.rb +7 -4
  10. data/lib/oai/client/resumable.rb +42 -0
  11. data/lib/oai/harvester/shell.rb +40 -41
  12. data/lib/oai/provider.rb +85 -67
  13. data/lib/oai/provider/metadata_format/oai_dc.rb +5 -6
  14. data/lib/oai/provider/model/activerecord_caching_wrapper.rb +23 -25
  15. data/lib/oai/provider/model/activerecord_wrapper.rb +99 -51
  16. data/lib/oai/provider/response.rb +33 -31
  17. data/lib/oai/provider/response/get_record.rb +7 -7
  18. data/lib/oai/provider/response/list_records.rb +5 -4
  19. data/lib/oai/provider/response/record_response.rb +14 -14
  20. data/test/activerecord_provider/config/connection.rb +8 -4
  21. data/test/activerecord_provider/database/{ar_migration.rb → 0001_oaipmh_tables.rb} +17 -12
  22. data/test/activerecord_provider/helpers/providers.rb +2 -3
  23. data/test/activerecord_provider/helpers/set_provider.rb +10 -22
  24. data/test/activerecord_provider/helpers/transactional_test_case.rb +34 -0
  25. data/test/activerecord_provider/models/dc_field.rb +4 -4
  26. data/test/activerecord_provider/models/dc_set.rb +3 -2
  27. data/test/activerecord_provider/models/exclusive_set_dc_field.rb +11 -0
  28. data/test/activerecord_provider/tc_ar_provider.rb +67 -28
  29. data/test/activerecord_provider/tc_ar_sets_provider.rb +104 -18
  30. data/test/activerecord_provider/tc_caching_paging_provider.rb +6 -10
  31. data/test/activerecord_provider/tc_simple_paging_provider.rb +7 -11
  32. data/test/activerecord_provider/test_helper.rb +10 -0
  33. data/test/client/helpers/provider.rb +44 -47
  34. data/test/client/helpers/test_wrapper.rb +4 -16
  35. data/test/client/tc_http_client.rb +90 -2
  36. data/test/client/tc_list_identifiers.rb +22 -3
  37. data/test/client/tc_list_records.rb +17 -4
  38. data/test/client/tc_list_sets.rb +17 -2
  39. data/test/provider/models.rb +32 -30
  40. data/test/provider/tc_exceptions.rb +30 -20
  41. data/test/provider/tc_functional_tokens.rb +11 -6
  42. data/test/provider/tc_provider.rb +58 -24
  43. data/test/provider/tc_resumption_tokens.rb +6 -6
  44. data/test/provider/tc_simple_provider.rb +51 -26
  45. data/test/provider/test_helper.rb +7 -0
  46. metadata +67 -128
  47. data/test/activerecord_provider/config/database.yml +0 -6
  48. data/test/activerecord_provider/database/oaipmhtest +0 -0
@@ -1,6 +1,6 @@
1
1
  require 'test_helper'
2
2
 
3
- class ActiveRecordSetProviderTest < Test::Unit::TestCase
3
+ class ActiveRecordSetProviderTest < TransactionalTestCase
4
4
 
5
5
  def test_list_sets
6
6
  doc = REXML::Document.new(@provider.list_sets)
@@ -8,51 +8,54 @@ class ActiveRecordSetProviderTest < Test::Unit::TestCase
8
8
  assert sets.size == 4
9
9
  assert sets[0].elements["//setName"].text == "Set A"
10
10
  end
11
-
11
+
12
12
  def test_set_a
13
- doc = REXML::Document.new(@provider.list_records(:set => "A"))
13
+ doc = REXML::Document.new(@provider.list_records(
14
+ :metadata_prefix => 'oai_dc', :set => "A"))
14
15
  assert_equal 20, doc.elements['OAI-PMH/ListRecords'].to_a.size
15
16
  end
16
-
17
+
17
18
  def test_set_b
18
- doc = REXML::Document.new(@provider.list_records(:set => "B"))
19
+ doc = REXML::Document.new(@provider.list_records(
20
+ :metadata_prefix => 'oai_dc', :set => "B"))
19
21
  assert_equal 10, doc.elements['OAI-PMH/ListRecords'].to_a.size
20
22
  end
21
-
23
+
22
24
  def test_set_ab
23
- doc = REXML::Document.new(@provider.list_records(:set => "A:B"))
25
+ doc = REXML::Document.new(@provider.list_records(
26
+ :metadata_prefix => 'oai_dc', :set => "A:B"))
24
27
  assert_equal 10, doc.elements['OAI-PMH/ListRecords'].to_a.size
25
28
  end
26
-
29
+
27
30
  def test_record_with_multiple_sets
28
31
  record = DCSet.find(:first, :conditions => "spec = 'C'").dc_fields.first
29
32
  assert_equal 2, record.sets.size
30
33
  end
31
34
 
35
+ def test_missing_set
36
+ assert_raise(OAI::NoMatchException) do
37
+ doc = REXML::Document.new(@provider.list_records(
38
+ :metadata_prefix => 'oai_dc', :set => "D"))
39
+ end
40
+ end
41
+
32
42
  def setup
33
43
  @provider = ARSetProvider.new
34
- ARLoader.load
35
44
  define_sets
36
45
  end
37
46
 
38
- def teardown
39
- ARLoader.unload
40
- DCSet.connection.execute("delete from dc_fields_dc_sets")
41
- DCSet.delete_all
42
- end
43
-
44
47
  def define_sets
45
48
  set_a = DCSet.create(:name => "Set A", :spec => "A")
46
49
  set_b = DCSet.create(:name => "Set B", :spec => "B")
47
50
  set_c = DCSet.create(:name => "Set C", :spec => "C")
48
51
  set_ab = DCSet.create(:name => "Set A:B", :spec => "A:B")
49
-
52
+
50
53
  next_id = 0
51
54
  DCField.find(:all, :limit => 10, :order => "id asc").each do |record|
52
55
  set_a.dc_fields << record
53
56
  next_id = record.id
54
57
  end
55
-
58
+
56
59
  DCField.find(:all, :limit => 10, :order => "id asc", :conditions => "id > #{next_id}").each do |record|
57
60
  set_b.dc_fields << record
58
61
  next_id = record.id
@@ -62,11 +65,94 @@ class ActiveRecordSetProviderTest < Test::Unit::TestCase
62
65
  set_ab.dc_fields << record
63
66
  next_id = record.id
64
67
  end
65
-
68
+
66
69
  DCField.find(:all, :limit => 10, :order => "id asc", :conditions => "id > #{next_id}").each do |record|
67
70
  set_a.dc_fields << record
68
71
  set_c.dc_fields << record
69
72
  next_id = record.id
70
73
  end
71
74
  end
75
+ end
76
+
77
+
78
+ class ActiveRecordExclusiveSetsProviderTest < TransactionalTestCase
79
+
80
+ def test_list_sets
81
+ doc = REXML::Document.new(@provider.list_sets)
82
+ sets = doc.elements["/OAI-PMH/ListSets"]
83
+ assert_equal 3, sets.size
84
+ assert_equal "Set A", sets[0].elements["//setName"].text
85
+ end
86
+
87
+ def test_set_a
88
+ doc = REXML::Document.new(@provider.list_records(
89
+ :metadata_prefix => 'oai_dc', :set => "A"))
90
+ assert_equal 20, doc.elements['OAI-PMH/ListRecords'].to_a.size
91
+ end
92
+
93
+ def test_set_b
94
+ doc = REXML::Document.new(@provider.list_records(
95
+ :metadata_prefix => 'oai_dc', :set => "B"))
96
+ assert_equal 10, doc.elements['OAI-PMH/ListRecords'].to_a.size
97
+ end
98
+
99
+ def test_set_ab
100
+ doc = REXML::Document.new(@provider.list_records(
101
+ :metadata_prefix => 'oai_dc', :set => "A:B"))
102
+ assert_equal 10, doc.elements['OAI-PMH/ListRecords'].to_a.size
103
+ end
104
+
105
+ def test_missing_set
106
+ assert_raise(OAI::NoMatchException) do
107
+ doc = REXML::Document.new(@provider.list_records(
108
+ :metadata_prefix => 'oai_dc', :set => "D"))
109
+ end
110
+ end
111
+
112
+ def setup
113
+ @provider = ARExclusiveSetProvider.new
114
+ define_sets
115
+ end
116
+
117
+ def define_sets
118
+ next_id = 0
119
+
120
+ ExclusiveSetDCField.find(:all, :limit => 10, :order => "id asc").each do |record|
121
+ record.set = "A"
122
+ record.save!
123
+ next_id = record.id
124
+ end
125
+
126
+ ExclusiveSetDCField.find(:all, :limit => 10, :order => "id asc", :conditions => "id > #{next_id}").each do |record|
127
+ record.set = "B"
128
+ record.save!
129
+ next_id = record.id
130
+ end
131
+
132
+ ExclusiveSetDCField.find(:all, :limit => 10, :order => "id asc", :conditions => "id > #{next_id}").each do |record|
133
+ record.set = "A:B"
134
+ record.save!
135
+ next_id = record.id
136
+ end
137
+
138
+ ExclusiveSetDCField.find(:all, :limit => 10, :order => "id asc", :conditions => "id > #{next_id}").each do |record|
139
+ record.set = "A"
140
+ record.save!
141
+ next_id = record.id
142
+ end
143
+ end
144
+
145
+ protected
146
+
147
+ def load_fixtures
148
+ fixtures = YAML.load_file(
149
+ File.join(File.dirname(__FILE__), 'fixtures', 'dc.yml')
150
+ )
151
+ disable_logging do
152
+ fixtures.keys.sort.each do |key|
153
+ ExclusiveSetDCField.create(fixtures[key])
154
+ end
155
+ end
156
+ end
157
+
72
158
  end
@@ -1,10 +1,10 @@
1
1
  require 'test_helper'
2
2
 
3
- class CachingPagingProviderTest < Test::Unit::TestCase
3
+ class CachingPagingProviderTest < TransactionalTestCase
4
4
  include REXML
5
-
5
+
6
6
  def test_full_harvest
7
- doc = Document.new(@provider.list_records)
7
+ doc = Document.new(@provider.list_records(:metadata_prefix => 'oai_dc'))
8
8
  assert_not_nil doc.elements["/OAI-PMH/ListRecords/resumptionToken"]
9
9
  assert_equal 26, doc.elements["/OAI-PMH/ListRecords"].size
10
10
  token = doc.elements["/OAI-PMH/ListRecords/resumptionToken"].text
@@ -20,7 +20,7 @@ class CachingPagingProviderTest < Test::Unit::TestCase
20
20
  assert_nil doc.elements["/OAI-PMH/ListRecords/resumptionToken"]
21
21
  assert_equal 25, doc.elements["/OAI-PMH/ListRecords"].size
22
22
  end
23
-
23
+
24
24
  def test_from_and_until
25
25
  first_id = DCField.find(:first, :order => "id asc").id
26
26
  DCField.update_all(['updated_at = ?', Time.parse("September 15 2005")],
@@ -31,6 +31,7 @@ class CachingPagingProviderTest < Test::Unit::TestCase
31
31
  # Should return 50 records broken into 2 groups of 25.
32
32
  doc = Document.new(
33
33
  @provider.list_records(
34
+ :metadata_prefix => 'oai_dc',
34
35
  :from => Time.parse("September 1 2005"),
35
36
  :until => Time.parse("November 30 2005"))
36
37
  )
@@ -44,11 +45,6 @@ class CachingPagingProviderTest < Test::Unit::TestCase
44
45
 
45
46
  def setup
46
47
  @provider = CachingResumptionProvider.new
47
- ARLoader.load
48
48
  end
49
-
50
- def teardown
51
- ARLoader.unload
52
- end
53
-
49
+
54
50
  end
@@ -1,10 +1,10 @@
1
1
  require 'test_helper'
2
2
 
3
- class SimpleResumptionProviderTest < Test::Unit::TestCase
3
+ class SimpleResumptionProviderTest < TransactionalTestCase
4
4
  include REXML
5
-
5
+
6
6
  def test_full_harvest
7
- doc = Document.new(@provider.list_records)
7
+ doc = Document.new(@provider.list_records(:metadata_prefix => 'oai_dc'))
8
8
  assert_not_nil doc.elements["/OAI-PMH/ListRecords/resumptionToken"]
9
9
  assert_equal 26, doc.elements["/OAI-PMH/ListRecords"].to_a.size
10
10
  token = doc.elements["/OAI-PMH/ListRecords/resumptionToken"].text
@@ -20,19 +20,20 @@ class SimpleResumptionProviderTest < Test::Unit::TestCase
20
20
  assert_nil doc.elements["/OAI-PMH/ListRecords/resumptionToken"]
21
21
  assert_equal 25, doc.elements["/OAI-PMH/ListRecords"].to_a.size
22
22
  end
23
-
23
+
24
24
  def test_from_and_until
25
25
  first_id = DCField.find(:first, :order => "id asc").id
26
26
  DCField.update_all(['updated_at = ?', Time.parse("September 15 2005")],
27
27
  "id < #{first_id + 25}")
28
28
  DCField.update_all(['updated_at = ?', Time.parse("November 1 2005")],
29
29
  "id <= #{first_id + 50} and id > #{first_id + 25}")
30
-
30
+
31
31
  total = DCField.count(:id, :conditions => ["updated_at >= ? AND updated_at <= ?", Time.parse("September 1 2005"), Time.parse("November 30 2005")])
32
32
 
33
33
  # Should return 50 records broken into 2 groups of 25.
34
34
  doc = Document.new(
35
35
  @provider.list_records(
36
+ :metadata_prefix => 'oai_dc',
36
37
  :from => Time.parse("September 1 2005"),
37
38
  :until => Time.parse("November 30 2005"))
38
39
  )
@@ -46,11 +47,6 @@ class SimpleResumptionProviderTest < Test::Unit::TestCase
46
47
 
47
48
  def setup
48
49
  @provider = SimpleResumptionProvider.new
49
- ARLoader.load
50
50
  end
51
-
52
- def teardown
53
- ARLoader.unload
54
- end
55
-
51
+
56
52
  end
@@ -1,4 +1,14 @@
1
1
  require 'rubygems'
2
+
3
+ if ENV['COVERAGE'] and RUBY_VERSION =~ /^1.9/
4
+ require 'simplecov'
5
+ require 'simplecov-rcov'
6
+
7
+ SimpleCov.formatter = SimpleCov::Formatter::RcovFormatter
8
+ SimpleCov.start
9
+ end
2
10
  require 'test/unit'
11
+ require File.dirname(__FILE__) + '/config/connection'
3
12
  require File.dirname(__FILE__) + '/helpers/providers'
4
13
  require File.dirname(__FILE__) + '/helpers/set_provider'
14
+ require File.dirname(__FILE__) + '/helpers/transactional_test_case'
@@ -8,61 +8,58 @@ class ComplexProvider < OAI::Provider::Base
8
8
  source_model ComplexModel.new(100)
9
9
  end
10
10
 
11
- class ProviderServer < WEBrick::HTTPServlet::AbstractServlet
12
- @@server = nil
13
-
14
- def initialize(server)
15
- super(server)
11
+ class ProviderServer
12
+
13
+ attr_reader :consumed, :server
14
+
15
+ def initialize(port, mount_point)
16
+ @consumed = []
16
17
  @provider = ComplexProvider.new
18
+ @server = WEBrick::HTTPServer.new(
19
+ :BindAddress => '127.0.0.1',
20
+ :Logger => WEBrick::Log.new('/dev/null'),
21
+ :AccessLog => [],
22
+ :Port => port)
23
+ @server.mount_proc(mount_point, server_proc)
17
24
  end
18
-
19
- def do_GET(req, res)
20
- begin
21
- res.body = @provider.process_request(req.query)
22
- res.status = 200
23
- res['Content-Type'] = 'text/xml'
24
- rescue => err
25
- puts err
26
- puts err.backtrace.join("\n")
27
- res.body = err.backtrace.join("\n")
28
- res.status = 500
29
- end
25
+
26
+ def port
27
+ @server.config[:Port]
30
28
  end
31
-
32
- def self.start(port)
33
- unless @@server
34
- @@server = WEBrick::HTTPServer.new(
35
- :BindAddress => '127.0.0.1',
36
- :Logger => WEBrick::Log.new('/dev/null'),
37
- :AccessLog => [],
38
- :Port => port)
39
- @@server.mount("/oai", ProviderServer)
40
29
 
41
- trap("INT") { @@server.shutdown }
42
- @@thread = Thread.new { @@server.start }
43
- puts "Starting Webrick/Provider on port[#{port}]"
44
- end
30
+ def start
31
+ @thread = Thread.new { @server.start }
45
32
  end
46
-
47
- def self.stop
48
- puts "Stopping Webrick/Provider"
49
- if @@thread
50
- @@thread.exit
51
- end
33
+
34
+ def stop
35
+ @thread.exit if @thread
52
36
  end
53
-
54
- def self.wrap(port = 3333)
37
+
38
+ def self.wrap(port = 3333, mount_point='/oai')
39
+ server = self.new(port, mount_point)
55
40
  begin
56
- start(port)
41
+ server.start
42
+ yield(server)
43
+ ensure
44
+ server.stop
45
+ end
46
+ end
57
47
 
58
- # Wait for startup
59
- sleep 2
60
-
61
- yield
48
+ protected
62
49
 
63
- ensure
64
- stop
50
+ def server_proc
51
+ Proc.new do |req, res|
52
+ begin
53
+ res.body = @provider.process_request(req.query)
54
+ res.status = 200
55
+ res['Content-Type'] = 'text/xml'
56
+ rescue => err
57
+ puts err
58
+ puts err.backtrace.join("\n")
59
+ res.body = err.backtrace.join("\n")
60
+ res.status = 500
61
+ end
65
62
  end
66
63
  end
67
-
68
- end
64
+
65
+ end
@@ -1,18 +1,6 @@
1
- module Test::Unit
2
- class AutoRunner
3
- alias_method :real_run, :run
4
-
5
- def run
6
- ProviderServer.wrap { real_run }
7
- end
8
-
9
- end
10
-
11
- end
12
-
13
- unless $provider_server_already_started
14
- $provider_server_already_started = true
15
- ProviderServer.start(3333)
16
- sleep 2
1
+ unless $provider_server
2
+ $provider_server = ProviderServer.new(3333, '/oai')
3
+ $provider_server.start
4
+ sleep 0.2
17
5
  end
18
6
 
@@ -1,4 +1,5 @@
1
1
  require 'test_helper'
2
+ require 'webrick'
2
3
 
3
4
  class HttpClientTest < Test::Unit::TestCase
4
5
 
@@ -6,7 +7,7 @@ class HttpClientTest < Test::Unit::TestCase
6
7
  oai_response = <<-eos
7
8
  <Identify>
8
9
  <repositoryName>Mock OAI Provider</repositoryName>
9
- <baseURL>http://nowhere.example.com</baseURL>
10
+ <baseURL>http://nowhere.example.com</baseURL>
10
11
  </Identify>
11
12
  eos
12
13
 
@@ -20,7 +21,94 @@ eos
20
21
 
21
22
  assert_kind_of OAI::IdentifyResponse, response
22
23
  assert_equal 'Mock OAI Provider [http://nowhere.example.com]', response.to_s
23
-
24
+
25
+ end
26
+
27
+ def test_http_client_handles_trailing_slash_redirects
28
+ # First, test that this works when mocking out Faraday client
29
+ oai_response = <<-eos
30
+ <Identify>
31
+ <repositoryName>Mock OAI Provider</repositoryName>
32
+ <baseURL>http://nowhere.example.com</baseURL>
33
+ </Identify>
34
+ eos
35
+
36
+ stubs = TrailingSlashAwareStubs.new do |stub|
37
+ stub.get('/oai/?verb=Identify') { [200, {}, oai_response] }
38
+ stub.get('/oai?verb=Identify') {
39
+ [301, {
40
+ 'Location' => 'http://localhost:3334/oai/?verb=Identify'
41
+ }, '']
42
+ }
43
+ end
44
+
45
+ faraday_stub = Faraday.new do |builder|
46
+ require 'faraday_middleware'
47
+ builder.use FaradayMiddleware::FollowRedirects
48
+ builder.adapter :test, stubs
49
+ end
50
+
51
+ client = OAI::Client.new 'http://localhost:3334/oai', :http => faraday_stub
52
+ response = client.identify
53
+
54
+ assert_kind_of OAI::IdentifyResponse, response
55
+ assert_equal 'Mock OAI Provider [http://nowhere.example.com]', response.to_s
56
+ assert_equal 2, stubs.consumed[:get].length
57
+ assert_equal stubs.consumed[:get].first.path, '/oai'
58
+ assert_equal stubs.consumed[:get].last.path, '/oai/'
59
+
60
+ # Now try it with a real server and default Faraday client
61
+ TrailingSlashProviderServer.wrap(3334) do |server|
62
+ client = OAI::Client.new "http://localhost:#{server.port}/oai"
63
+ response = client.identify
64
+
65
+ assert_kind_of OAI::IdentifyResponse, response
66
+ assert_equal 'Complex Provider [http://localhost]', response.to_s
67
+ assert_equal 2, server.consumed.length
68
+ assert_equal server.consumed.first.path, '/oai'
69
+ assert_equal server.consumed.last.path, '/oai/'
70
+ end
71
+ end
72
+
73
+ private
74
+
75
+ class TrailingSlashProviderServer < ProviderServer
76
+ def server_proc
77
+ Proc.new do |req, res|
78
+ @consumed << req
79
+ case req.path
80
+ when "/oai/"
81
+ begin
82
+ res.body = @provider.process_request(req.query)
83
+ res.status = 200
84
+ res['Content-Type'] = 'text/xml'
85
+ rescue => err
86
+ puts err
87
+ puts err.backtrace.join("\n")
88
+ res.body = err.backtrace.join("\n")
89
+ res.status = 500
90
+ end
91
+ else
92
+ res.body = ''
93
+ res.status = 301
94
+ res['Location'] = "http://localhost:#{port}/oai/?#{req.query_string}"
95
+ end
96
+ res
97
+ end
98
+ end
99
+ end
100
+
101
+ class TrailingSlashAwareStubs < Faraday::Adapter::Test::Stubs
102
+ attr_reader :consumed
103
+
104
+ # ensure leading, but not trailing slash
105
+ def normalize_path(path)
106
+ path = '/' + path if path.index('/') != 0
107
+ #path = path.sub('?', '/?')
108
+ #path = path + '/' unless $&
109
+ path.gsub('//', '/')
110
+ end
111
+
24
112
  end
25
113
  end
26
114