oai 0.2.1 → 0.3.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (48) hide show
  1. data/README.md +28 -23
  2. data/Rakefile +14 -40
  3. data/examples/providers/dublin_core.rb +63 -63
  4. data/lib/oai/client.rb +131 -97
  5. data/lib/oai/client/list_identifiers.rb +1 -0
  6. data/lib/oai/client/list_records.rb +6 -5
  7. data/lib/oai/client/list_sets.rb +6 -5
  8. data/lib/oai/client/record.rb +6 -7
  9. data/lib/oai/client/response.rb +7 -4
  10. data/lib/oai/client/resumable.rb +42 -0
  11. data/lib/oai/harvester/shell.rb +40 -41
  12. data/lib/oai/provider.rb +85 -67
  13. data/lib/oai/provider/metadata_format/oai_dc.rb +5 -6
  14. data/lib/oai/provider/model/activerecord_caching_wrapper.rb +23 -25
  15. data/lib/oai/provider/model/activerecord_wrapper.rb +99 -51
  16. data/lib/oai/provider/response.rb +33 -31
  17. data/lib/oai/provider/response/get_record.rb +7 -7
  18. data/lib/oai/provider/response/list_records.rb +5 -4
  19. data/lib/oai/provider/response/record_response.rb +14 -14
  20. data/test/activerecord_provider/config/connection.rb +8 -4
  21. data/test/activerecord_provider/database/{ar_migration.rb → 0001_oaipmh_tables.rb} +17 -12
  22. data/test/activerecord_provider/helpers/providers.rb +2 -3
  23. data/test/activerecord_provider/helpers/set_provider.rb +10 -22
  24. data/test/activerecord_provider/helpers/transactional_test_case.rb +34 -0
  25. data/test/activerecord_provider/models/dc_field.rb +4 -4
  26. data/test/activerecord_provider/models/dc_set.rb +3 -2
  27. data/test/activerecord_provider/models/exclusive_set_dc_field.rb +11 -0
  28. data/test/activerecord_provider/tc_ar_provider.rb +67 -28
  29. data/test/activerecord_provider/tc_ar_sets_provider.rb +104 -18
  30. data/test/activerecord_provider/tc_caching_paging_provider.rb +6 -10
  31. data/test/activerecord_provider/tc_simple_paging_provider.rb +7 -11
  32. data/test/activerecord_provider/test_helper.rb +10 -0
  33. data/test/client/helpers/provider.rb +44 -47
  34. data/test/client/helpers/test_wrapper.rb +4 -16
  35. data/test/client/tc_http_client.rb +90 -2
  36. data/test/client/tc_list_identifiers.rb +22 -3
  37. data/test/client/tc_list_records.rb +17 -4
  38. data/test/client/tc_list_sets.rb +17 -2
  39. data/test/provider/models.rb +32 -30
  40. data/test/provider/tc_exceptions.rb +30 -20
  41. data/test/provider/tc_functional_tokens.rb +11 -6
  42. data/test/provider/tc_provider.rb +58 -24
  43. data/test/provider/tc_resumption_tokens.rb +6 -6
  44. data/test/provider/tc_simple_provider.rb +51 -26
  45. data/test/provider/test_helper.rb +7 -0
  46. metadata +67 -128
  47. data/test/activerecord_provider/config/database.yml +0 -6
  48. data/test/activerecord_provider/database/oaipmhtest +0 -0
@@ -1,6 +1,6 @@
1
1
  require 'test_helper'
2
2
 
3
- class ActiveRecordSetProviderTest < Test::Unit::TestCase
3
+ class ActiveRecordSetProviderTest < TransactionalTestCase
4
4
 
5
5
  def test_list_sets
6
6
  doc = REXML::Document.new(@provider.list_sets)
@@ -8,51 +8,54 @@ class ActiveRecordSetProviderTest < Test::Unit::TestCase
8
8
  assert sets.size == 4
9
9
  assert sets[0].elements["//setName"].text == "Set A"
10
10
  end
11
-
11
+
12
12
  def test_set_a
13
- doc = REXML::Document.new(@provider.list_records(:set => "A"))
13
+ doc = REXML::Document.new(@provider.list_records(
14
+ :metadata_prefix => 'oai_dc', :set => "A"))
14
15
  assert_equal 20, doc.elements['OAI-PMH/ListRecords'].to_a.size
15
16
  end
16
-
17
+
17
18
  def test_set_b
18
- doc = REXML::Document.new(@provider.list_records(:set => "B"))
19
+ doc = REXML::Document.new(@provider.list_records(
20
+ :metadata_prefix => 'oai_dc', :set => "B"))
19
21
  assert_equal 10, doc.elements['OAI-PMH/ListRecords'].to_a.size
20
22
  end
21
-
23
+
22
24
  def test_set_ab
23
- doc = REXML::Document.new(@provider.list_records(:set => "A:B"))
25
+ doc = REXML::Document.new(@provider.list_records(
26
+ :metadata_prefix => 'oai_dc', :set => "A:B"))
24
27
  assert_equal 10, doc.elements['OAI-PMH/ListRecords'].to_a.size
25
28
  end
26
-
29
+
27
30
  def test_record_with_multiple_sets
28
31
  record = DCSet.find(:first, :conditions => "spec = 'C'").dc_fields.first
29
32
  assert_equal 2, record.sets.size
30
33
  end
31
34
 
35
+ def test_missing_set
36
+ assert_raise(OAI::NoMatchException) do
37
+ doc = REXML::Document.new(@provider.list_records(
38
+ :metadata_prefix => 'oai_dc', :set => "D"))
39
+ end
40
+ end
41
+
32
42
  def setup
33
43
  @provider = ARSetProvider.new
34
- ARLoader.load
35
44
  define_sets
36
45
  end
37
46
 
38
- def teardown
39
- ARLoader.unload
40
- DCSet.connection.execute("delete from dc_fields_dc_sets")
41
- DCSet.delete_all
42
- end
43
-
44
47
  def define_sets
45
48
  set_a = DCSet.create(:name => "Set A", :spec => "A")
46
49
  set_b = DCSet.create(:name => "Set B", :spec => "B")
47
50
  set_c = DCSet.create(:name => "Set C", :spec => "C")
48
51
  set_ab = DCSet.create(:name => "Set A:B", :spec => "A:B")
49
-
52
+
50
53
  next_id = 0
51
54
  DCField.find(:all, :limit => 10, :order => "id asc").each do |record|
52
55
  set_a.dc_fields << record
53
56
  next_id = record.id
54
57
  end
55
-
58
+
56
59
  DCField.find(:all, :limit => 10, :order => "id asc", :conditions => "id > #{next_id}").each do |record|
57
60
  set_b.dc_fields << record
58
61
  next_id = record.id
@@ -62,11 +65,94 @@ class ActiveRecordSetProviderTest < Test::Unit::TestCase
62
65
  set_ab.dc_fields << record
63
66
  next_id = record.id
64
67
  end
65
-
68
+
66
69
  DCField.find(:all, :limit => 10, :order => "id asc", :conditions => "id > #{next_id}").each do |record|
67
70
  set_a.dc_fields << record
68
71
  set_c.dc_fields << record
69
72
  next_id = record.id
70
73
  end
71
74
  end
75
+ end
76
+
77
+
78
+ class ActiveRecordExclusiveSetsProviderTest < TransactionalTestCase
79
+
80
+ def test_list_sets
81
+ doc = REXML::Document.new(@provider.list_sets)
82
+ sets = doc.elements["/OAI-PMH/ListSets"]
83
+ assert_equal 3, sets.size
84
+ assert_equal "Set A", sets[0].elements["//setName"].text
85
+ end
86
+
87
+ def test_set_a
88
+ doc = REXML::Document.new(@provider.list_records(
89
+ :metadata_prefix => 'oai_dc', :set => "A"))
90
+ assert_equal 20, doc.elements['OAI-PMH/ListRecords'].to_a.size
91
+ end
92
+
93
+ def test_set_b
94
+ doc = REXML::Document.new(@provider.list_records(
95
+ :metadata_prefix => 'oai_dc', :set => "B"))
96
+ assert_equal 10, doc.elements['OAI-PMH/ListRecords'].to_a.size
97
+ end
98
+
99
+ def test_set_ab
100
+ doc = REXML::Document.new(@provider.list_records(
101
+ :metadata_prefix => 'oai_dc', :set => "A:B"))
102
+ assert_equal 10, doc.elements['OAI-PMH/ListRecords'].to_a.size
103
+ end
104
+
105
+ def test_missing_set
106
+ assert_raise(OAI::NoMatchException) do
107
+ doc = REXML::Document.new(@provider.list_records(
108
+ :metadata_prefix => 'oai_dc', :set => "D"))
109
+ end
110
+ end
111
+
112
+ def setup
113
+ @provider = ARExclusiveSetProvider.new
114
+ define_sets
115
+ end
116
+
117
+ def define_sets
118
+ next_id = 0
119
+
120
+ ExclusiveSetDCField.find(:all, :limit => 10, :order => "id asc").each do |record|
121
+ record.set = "A"
122
+ record.save!
123
+ next_id = record.id
124
+ end
125
+
126
+ ExclusiveSetDCField.find(:all, :limit => 10, :order => "id asc", :conditions => "id > #{next_id}").each do |record|
127
+ record.set = "B"
128
+ record.save!
129
+ next_id = record.id
130
+ end
131
+
132
+ ExclusiveSetDCField.find(:all, :limit => 10, :order => "id asc", :conditions => "id > #{next_id}").each do |record|
133
+ record.set = "A:B"
134
+ record.save!
135
+ next_id = record.id
136
+ end
137
+
138
+ ExclusiveSetDCField.find(:all, :limit => 10, :order => "id asc", :conditions => "id > #{next_id}").each do |record|
139
+ record.set = "A"
140
+ record.save!
141
+ next_id = record.id
142
+ end
143
+ end
144
+
145
+ protected
146
+
147
+ def load_fixtures
148
+ fixtures = YAML.load_file(
149
+ File.join(File.dirname(__FILE__), 'fixtures', 'dc.yml')
150
+ )
151
+ disable_logging do
152
+ fixtures.keys.sort.each do |key|
153
+ ExclusiveSetDCField.create(fixtures[key])
154
+ end
155
+ end
156
+ end
157
+
72
158
  end
@@ -1,10 +1,10 @@
1
1
  require 'test_helper'
2
2
 
3
- class CachingPagingProviderTest < Test::Unit::TestCase
3
+ class CachingPagingProviderTest < TransactionalTestCase
4
4
  include REXML
5
-
5
+
6
6
  def test_full_harvest
7
- doc = Document.new(@provider.list_records)
7
+ doc = Document.new(@provider.list_records(:metadata_prefix => 'oai_dc'))
8
8
  assert_not_nil doc.elements["/OAI-PMH/ListRecords/resumptionToken"]
9
9
  assert_equal 26, doc.elements["/OAI-PMH/ListRecords"].size
10
10
  token = doc.elements["/OAI-PMH/ListRecords/resumptionToken"].text
@@ -20,7 +20,7 @@ class CachingPagingProviderTest < Test::Unit::TestCase
20
20
  assert_nil doc.elements["/OAI-PMH/ListRecords/resumptionToken"]
21
21
  assert_equal 25, doc.elements["/OAI-PMH/ListRecords"].size
22
22
  end
23
-
23
+
24
24
  def test_from_and_until
25
25
  first_id = DCField.find(:first, :order => "id asc").id
26
26
  DCField.update_all(['updated_at = ?', Time.parse("September 15 2005")],
@@ -31,6 +31,7 @@ class CachingPagingProviderTest < Test::Unit::TestCase
31
31
  # Should return 50 records broken into 2 groups of 25.
32
32
  doc = Document.new(
33
33
  @provider.list_records(
34
+ :metadata_prefix => 'oai_dc',
34
35
  :from => Time.parse("September 1 2005"),
35
36
  :until => Time.parse("November 30 2005"))
36
37
  )
@@ -44,11 +45,6 @@ class CachingPagingProviderTest < Test::Unit::TestCase
44
45
 
45
46
  def setup
46
47
  @provider = CachingResumptionProvider.new
47
- ARLoader.load
48
48
  end
49
-
50
- def teardown
51
- ARLoader.unload
52
- end
53
-
49
+
54
50
  end
@@ -1,10 +1,10 @@
1
1
  require 'test_helper'
2
2
 
3
- class SimpleResumptionProviderTest < Test::Unit::TestCase
3
+ class SimpleResumptionProviderTest < TransactionalTestCase
4
4
  include REXML
5
-
5
+
6
6
  def test_full_harvest
7
- doc = Document.new(@provider.list_records)
7
+ doc = Document.new(@provider.list_records(:metadata_prefix => 'oai_dc'))
8
8
  assert_not_nil doc.elements["/OAI-PMH/ListRecords/resumptionToken"]
9
9
  assert_equal 26, doc.elements["/OAI-PMH/ListRecords"].to_a.size
10
10
  token = doc.elements["/OAI-PMH/ListRecords/resumptionToken"].text
@@ -20,19 +20,20 @@ class SimpleResumptionProviderTest < Test::Unit::TestCase
20
20
  assert_nil doc.elements["/OAI-PMH/ListRecords/resumptionToken"]
21
21
  assert_equal 25, doc.elements["/OAI-PMH/ListRecords"].to_a.size
22
22
  end
23
-
23
+
24
24
  def test_from_and_until
25
25
  first_id = DCField.find(:first, :order => "id asc").id
26
26
  DCField.update_all(['updated_at = ?', Time.parse("September 15 2005")],
27
27
  "id < #{first_id + 25}")
28
28
  DCField.update_all(['updated_at = ?', Time.parse("November 1 2005")],
29
29
  "id <= #{first_id + 50} and id > #{first_id + 25}")
30
-
30
+
31
31
  total = DCField.count(:id, :conditions => ["updated_at >= ? AND updated_at <= ?", Time.parse("September 1 2005"), Time.parse("November 30 2005")])
32
32
 
33
33
  # Should return 50 records broken into 2 groups of 25.
34
34
  doc = Document.new(
35
35
  @provider.list_records(
36
+ :metadata_prefix => 'oai_dc',
36
37
  :from => Time.parse("September 1 2005"),
37
38
  :until => Time.parse("November 30 2005"))
38
39
  )
@@ -46,11 +47,6 @@ class SimpleResumptionProviderTest < Test::Unit::TestCase
46
47
 
47
48
  def setup
48
49
  @provider = SimpleResumptionProvider.new
49
- ARLoader.load
50
50
  end
51
-
52
- def teardown
53
- ARLoader.unload
54
- end
55
-
51
+
56
52
  end
@@ -1,4 +1,14 @@
1
1
  require 'rubygems'
2
+
3
+ if ENV['COVERAGE'] and RUBY_VERSION =~ /^1.9/
4
+ require 'simplecov'
5
+ require 'simplecov-rcov'
6
+
7
+ SimpleCov.formatter = SimpleCov::Formatter::RcovFormatter
8
+ SimpleCov.start
9
+ end
2
10
  require 'test/unit'
11
+ require File.dirname(__FILE__) + '/config/connection'
3
12
  require File.dirname(__FILE__) + '/helpers/providers'
4
13
  require File.dirname(__FILE__) + '/helpers/set_provider'
14
+ require File.dirname(__FILE__) + '/helpers/transactional_test_case'
@@ -8,61 +8,58 @@ class ComplexProvider < OAI::Provider::Base
8
8
  source_model ComplexModel.new(100)
9
9
  end
10
10
 
11
- class ProviderServer < WEBrick::HTTPServlet::AbstractServlet
12
- @@server = nil
13
-
14
- def initialize(server)
15
- super(server)
11
+ class ProviderServer
12
+
13
+ attr_reader :consumed, :server
14
+
15
+ def initialize(port, mount_point)
16
+ @consumed = []
16
17
  @provider = ComplexProvider.new
18
+ @server = WEBrick::HTTPServer.new(
19
+ :BindAddress => '127.0.0.1',
20
+ :Logger => WEBrick::Log.new('/dev/null'),
21
+ :AccessLog => [],
22
+ :Port => port)
23
+ @server.mount_proc(mount_point, server_proc)
17
24
  end
18
-
19
- def do_GET(req, res)
20
- begin
21
- res.body = @provider.process_request(req.query)
22
- res.status = 200
23
- res['Content-Type'] = 'text/xml'
24
- rescue => err
25
- puts err
26
- puts err.backtrace.join("\n")
27
- res.body = err.backtrace.join("\n")
28
- res.status = 500
29
- end
25
+
26
+ def port
27
+ @server.config[:Port]
30
28
  end
31
-
32
- def self.start(port)
33
- unless @@server
34
- @@server = WEBrick::HTTPServer.new(
35
- :BindAddress => '127.0.0.1',
36
- :Logger => WEBrick::Log.new('/dev/null'),
37
- :AccessLog => [],
38
- :Port => port)
39
- @@server.mount("/oai", ProviderServer)
40
29
 
41
- trap("INT") { @@server.shutdown }
42
- @@thread = Thread.new { @@server.start }
43
- puts "Starting Webrick/Provider on port[#{port}]"
44
- end
30
+ def start
31
+ @thread = Thread.new { @server.start }
45
32
  end
46
-
47
- def self.stop
48
- puts "Stopping Webrick/Provider"
49
- if @@thread
50
- @@thread.exit
51
- end
33
+
34
+ def stop
35
+ @thread.exit if @thread
52
36
  end
53
-
54
- def self.wrap(port = 3333)
37
+
38
+ def self.wrap(port = 3333, mount_point='/oai')
39
+ server = self.new(port, mount_point)
55
40
  begin
56
- start(port)
41
+ server.start
42
+ yield(server)
43
+ ensure
44
+ server.stop
45
+ end
46
+ end
57
47
 
58
- # Wait for startup
59
- sleep 2
60
-
61
- yield
48
+ protected
62
49
 
63
- ensure
64
- stop
50
+ def server_proc
51
+ Proc.new do |req, res|
52
+ begin
53
+ res.body = @provider.process_request(req.query)
54
+ res.status = 200
55
+ res['Content-Type'] = 'text/xml'
56
+ rescue => err
57
+ puts err
58
+ puts err.backtrace.join("\n")
59
+ res.body = err.backtrace.join("\n")
60
+ res.status = 500
61
+ end
65
62
  end
66
63
  end
67
-
68
- end
64
+
65
+ end
@@ -1,18 +1,6 @@
1
- module Test::Unit
2
- class AutoRunner
3
- alias_method :real_run, :run
4
-
5
- def run
6
- ProviderServer.wrap { real_run }
7
- end
8
-
9
- end
10
-
11
- end
12
-
13
- unless $provider_server_already_started
14
- $provider_server_already_started = true
15
- ProviderServer.start(3333)
16
- sleep 2
1
+ unless $provider_server
2
+ $provider_server = ProviderServer.new(3333, '/oai')
3
+ $provider_server.start
4
+ sleep 0.2
17
5
  end
18
6
 
@@ -1,4 +1,5 @@
1
1
  require 'test_helper'
2
+ require 'webrick'
2
3
 
3
4
  class HttpClientTest < Test::Unit::TestCase
4
5
 
@@ -6,7 +7,7 @@ class HttpClientTest < Test::Unit::TestCase
6
7
  oai_response = <<-eos
7
8
  <Identify>
8
9
  <repositoryName>Mock OAI Provider</repositoryName>
9
- <baseURL>http://nowhere.example.com</baseURL>
10
+ <baseURL>http://nowhere.example.com</baseURL>
10
11
  </Identify>
11
12
  eos
12
13
 
@@ -20,7 +21,94 @@ eos
20
21
 
21
22
  assert_kind_of OAI::IdentifyResponse, response
22
23
  assert_equal 'Mock OAI Provider [http://nowhere.example.com]', response.to_s
23
-
24
+
25
+ end
26
+
27
+ def test_http_client_handles_trailing_slash_redirects
28
+ # First, test that this works when mocking out Faraday client
29
+ oai_response = <<-eos
30
+ <Identify>
31
+ <repositoryName>Mock OAI Provider</repositoryName>
32
+ <baseURL>http://nowhere.example.com</baseURL>
33
+ </Identify>
34
+ eos
35
+
36
+ stubs = TrailingSlashAwareStubs.new do |stub|
37
+ stub.get('/oai/?verb=Identify') { [200, {}, oai_response] }
38
+ stub.get('/oai?verb=Identify') {
39
+ [301, {
40
+ 'Location' => 'http://localhost:3334/oai/?verb=Identify'
41
+ }, '']
42
+ }
43
+ end
44
+
45
+ faraday_stub = Faraday.new do |builder|
46
+ require 'faraday_middleware'
47
+ builder.use FaradayMiddleware::FollowRedirects
48
+ builder.adapter :test, stubs
49
+ end
50
+
51
+ client = OAI::Client.new 'http://localhost:3334/oai', :http => faraday_stub
52
+ response = client.identify
53
+
54
+ assert_kind_of OAI::IdentifyResponse, response
55
+ assert_equal 'Mock OAI Provider [http://nowhere.example.com]', response.to_s
56
+ assert_equal 2, stubs.consumed[:get].length
57
+ assert_equal stubs.consumed[:get].first.path, '/oai'
58
+ assert_equal stubs.consumed[:get].last.path, '/oai/'
59
+
60
+ # Now try it with a real server and default Faraday client
61
+ TrailingSlashProviderServer.wrap(3334) do |server|
62
+ client = OAI::Client.new "http://localhost:#{server.port}/oai"
63
+ response = client.identify
64
+
65
+ assert_kind_of OAI::IdentifyResponse, response
66
+ assert_equal 'Complex Provider [http://localhost]', response.to_s
67
+ assert_equal 2, server.consumed.length
68
+ assert_equal server.consumed.first.path, '/oai'
69
+ assert_equal server.consumed.last.path, '/oai/'
70
+ end
71
+ end
72
+
73
+ private
74
+
75
+ class TrailingSlashProviderServer < ProviderServer
76
+ def server_proc
77
+ Proc.new do |req, res|
78
+ @consumed << req
79
+ case req.path
80
+ when "/oai/"
81
+ begin
82
+ res.body = @provider.process_request(req.query)
83
+ res.status = 200
84
+ res['Content-Type'] = 'text/xml'
85
+ rescue => err
86
+ puts err
87
+ puts err.backtrace.join("\n")
88
+ res.body = err.backtrace.join("\n")
89
+ res.status = 500
90
+ end
91
+ else
92
+ res.body = ''
93
+ res.status = 301
94
+ res['Location'] = "http://localhost:#{port}/oai/?#{req.query_string}"
95
+ end
96
+ res
97
+ end
98
+ end
99
+ end
100
+
101
+ class TrailingSlashAwareStubs < Faraday::Adapter::Test::Stubs
102
+ attr_reader :consumed
103
+
104
+ # ensure leading, but not trailing slash
105
+ def normalize_path(path)
106
+ path = '/' + path if path.index('/') != 0
107
+ #path = path.sub('?', '/?')
108
+ #path = path + '/' unless $&
109
+ path.gsub('//', '/')
110
+ end
111
+
24
112
  end
25
113
  end
26
114