oai 0.2.1 → 0.3.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (48) hide show
  1. data/README.md +28 -23
  2. data/Rakefile +14 -40
  3. data/examples/providers/dublin_core.rb +63 -63
  4. data/lib/oai/client.rb +131 -97
  5. data/lib/oai/client/list_identifiers.rb +1 -0
  6. data/lib/oai/client/list_records.rb +6 -5
  7. data/lib/oai/client/list_sets.rb +6 -5
  8. data/lib/oai/client/record.rb +6 -7
  9. data/lib/oai/client/response.rb +7 -4
  10. data/lib/oai/client/resumable.rb +42 -0
  11. data/lib/oai/harvester/shell.rb +40 -41
  12. data/lib/oai/provider.rb +85 -67
  13. data/lib/oai/provider/metadata_format/oai_dc.rb +5 -6
  14. data/lib/oai/provider/model/activerecord_caching_wrapper.rb +23 -25
  15. data/lib/oai/provider/model/activerecord_wrapper.rb +99 -51
  16. data/lib/oai/provider/response.rb +33 -31
  17. data/lib/oai/provider/response/get_record.rb +7 -7
  18. data/lib/oai/provider/response/list_records.rb +5 -4
  19. data/lib/oai/provider/response/record_response.rb +14 -14
  20. data/test/activerecord_provider/config/connection.rb +8 -4
  21. data/test/activerecord_provider/database/{ar_migration.rb → 0001_oaipmh_tables.rb} +17 -12
  22. data/test/activerecord_provider/helpers/providers.rb +2 -3
  23. data/test/activerecord_provider/helpers/set_provider.rb +10 -22
  24. data/test/activerecord_provider/helpers/transactional_test_case.rb +34 -0
  25. data/test/activerecord_provider/models/dc_field.rb +4 -4
  26. data/test/activerecord_provider/models/dc_set.rb +3 -2
  27. data/test/activerecord_provider/models/exclusive_set_dc_field.rb +11 -0
  28. data/test/activerecord_provider/tc_ar_provider.rb +67 -28
  29. data/test/activerecord_provider/tc_ar_sets_provider.rb +104 -18
  30. data/test/activerecord_provider/tc_caching_paging_provider.rb +6 -10
  31. data/test/activerecord_provider/tc_simple_paging_provider.rb +7 -11
  32. data/test/activerecord_provider/test_helper.rb +10 -0
  33. data/test/client/helpers/provider.rb +44 -47
  34. data/test/client/helpers/test_wrapper.rb +4 -16
  35. data/test/client/tc_http_client.rb +90 -2
  36. data/test/client/tc_list_identifiers.rb +22 -3
  37. data/test/client/tc_list_records.rb +17 -4
  38. data/test/client/tc_list_sets.rb +17 -2
  39. data/test/provider/models.rb +32 -30
  40. data/test/provider/tc_exceptions.rb +30 -20
  41. data/test/provider/tc_functional_tokens.rb +11 -6
  42. data/test/provider/tc_provider.rb +58 -24
  43. data/test/provider/tc_resumption_tokens.rb +6 -6
  44. data/test/provider/tc_simple_provider.rb +51 -26
  45. data/test/provider/test_helper.rb +7 -0
  46. metadata +67 -128
  47. data/test/activerecord_provider/config/database.yml +0 -6
  48. data/test/activerecord_provider/database/oaipmhtest +0 -0
@@ -1,8 +1,8 @@
1
1
  module OAI::Provider::Response
2
-
2
+
3
3
  class GetRecord < RecordResponse
4
- required_parameters :identifier
5
-
4
+ required_parameters :identifier, :metadata_prefix
5
+
6
6
  def to_xml
7
7
  id = extract_identifier(options.delete(:identifier))
8
8
  unless record = provider.model.find(id, options)
@@ -11,7 +11,7 @@ module OAI::Provider::Response
11
11
 
12
12
  response do |r|
13
13
  r.GetRecord do
14
- r.record do
14
+ r.record do
15
15
  header_for record
16
16
  data_for record unless deleted?(record)
17
17
  about_for record unless deleted?(record)
@@ -19,9 +19,9 @@ module OAI::Provider::Response
19
19
  end
20
20
  end
21
21
  end
22
-
22
+
23
23
  end
24
24
 
25
25
  end
26
-
27
-
26
+
27
+
@@ -1,7 +1,8 @@
1
1
  module OAI::Provider::Response
2
2
 
3
3
  class ListRecords < RecordResponse
4
-
4
+ required_parameters :metadata_prefix
5
+
5
6
  def to_xml
6
7
  result = provider.model.find(:all, options)
7
8
  # result may be an array of records, or a partial result
@@ -27,8 +28,8 @@ module OAI::Provider::Response
27
28
  end
28
29
  end
29
30
  end
30
-
31
+
31
32
  end
32
-
33
+
33
34
  end
34
-
35
+
@@ -2,16 +2,16 @@ module OAI::Provider::Response
2
2
  class RecordResponse < Base
3
3
  def self.inherited(klass)
4
4
  klass.valid_parameters :metadata_prefix, :from, :until, :set
5
- klass.default_parameters :metadata_prefix => "oai_dc",
5
+ klass.default_parameters :metadata_prefix => "oai_dc",
6
6
  :from => Proc.new {|x| Time.parse(x.provider.model.earliest.to_s) }, #-- OAI 2.0 hack - UTC
7
7
  :until => Proc.new {|x| Time.parse(x.provider.model.latest.to_s) } #-- OAI 2.0 hack - UTC
8
8
  end
9
-
9
+
10
10
  # emit record header
11
11
  def header_for(record)
12
12
  param = Hash.new
13
13
  param[:status] = 'deleted' if deleted?(record)
14
- @builder.header param do
14
+ @builder.header param do
15
15
  @builder.identifier identifier_for(record)
16
16
  @builder.datestamp timestamp_for(record)
17
17
  sets_for(record).each do |set|
@@ -33,7 +33,7 @@ module OAI::Provider::Response
33
33
  return unless provider.model.respond_to? :about
34
34
 
35
35
  about = provider.model.about(record)
36
- return if about.nil?
36
+ return if about.nil?
37
37
 
38
38
  unless about.is_a? Array
39
39
  about = [about]
@@ -42,43 +42,43 @@ module OAI::Provider::Response
42
42
  about.each do |a|
43
43
  @builder.about do
44
44
  @builder.target! << a
45
- end
45
+ end
46
46
  end
47
47
  end
48
-
48
+
49
49
  private
50
-
50
+
51
51
  def identifier_for(record)
52
52
  "#{provider.prefix}/#{record.id}"
53
53
  end
54
-
54
+
55
55
  def timestamp_for(record)
56
56
  record.send(provider.model.timestamp_field).utc.xmlschema
57
57
  end
58
-
58
+
59
59
  def sets_for(record)
60
60
  return [] unless record.respond_to?(:sets) and record.sets
61
61
  record.sets.respond_to?(:each) ? record.sets : [record.sets]
62
62
  end
63
-
63
+
64
64
  def requested_format
65
- format =
65
+ format =
66
66
  if options[:metadata_prefix]
67
67
  options[:metadata_prefix]
68
68
  elsif options[:resumption_token]
69
69
  OAI::Provider::ResumptionToken.extract_format(options[:resumption_token])
70
70
  end
71
71
  raise OAI::FormatException.new unless provider.format_supported?(format)
72
-
72
+
73
73
  format
74
74
  end
75
-
75
+
76
76
  def deleted?(record)
77
77
  return record.deleted? if record.respond_to?(:deleted?)
78
78
  return record.deleted if record.respond_to?(:deleted)
79
79
  return record.deleted_at if record.respond_to?(:deleted_at)
80
80
  false
81
81
  end
82
-
82
+
83
83
  end
84
84
  end
@@ -1,5 +1,9 @@
1
+ require 'active_record'
2
+ require 'logger'
3
+
1
4
  # Configure AR connection
2
- conn_info = YAML.load_file(
3
- File.join(File.dirname(__FILE__), "database.yml")
4
- )
5
- ActiveRecord::Base.establish_connection(conn_info)
5
+ #ActiveRecord::Base.logger = Logger.new(STDOUT)
6
+ ActiveRecord::Migration.verbose = false
7
+ ActiveRecord::Base.establish_connection :adapter => "sqlite3",
8
+ :database => ":memory:"
9
+ ActiveRecord::Migrator.up File.join(File.dirname(__FILE__), '..', 'database')
@@ -1,18 +1,16 @@
1
- ActiveRecord::Migration.verbose = false
2
-
3
- class OAIPMHTables < ActiveRecord::Migration
1
+ class OaipmhTables < ActiveRecord::Migration
4
2
  def self.up
5
- create_table :oai_tokens, :force => true do |t|
3
+ create_table :oai_tokens do |t|
6
4
  t.column :token, :string, :null => false
7
5
  t.column :created_at, :timestamp
8
6
  end
9
-
10
- create_table :oai_entries, :force => true do |t|
7
+
8
+ create_table :oai_entries do |t|
11
9
  t.column :record_id, :integer, :null => false
12
10
  t.column :oai_token_id, :integer, :null => false
13
11
  end
14
-
15
- create_table :dc_fields, :force => true do |t|
12
+
13
+ dc_fields = proc do |t|
16
14
  t.column :title, :string
17
15
  t.column :creator, :string
18
16
  t.column :subject, :string
@@ -31,13 +29,20 @@ class OAIPMHTables < ActiveRecord::Migration
31
29
  t.column :created_at, :datetime
32
30
  t.column :deleted, :boolean, :default => false
33
31
  end
34
-
35
- create_table :dc_fields_dc_sets, :force => true, :id => false do |t|
32
+
33
+ create_table :exclusive_set_dc_fields do |t|
34
+ dc_fields.call(t)
35
+ t.column :set, :string
36
+ end
37
+
38
+ create_table :dc_fields, &dc_fields
39
+
40
+ create_table :dc_fields_dc_sets, :id => false do |t|
36
41
  t.column :dc_field_id, :integer
37
42
  t.column :dc_set_id, :integer
38
43
  end
39
-
40
- create_table :dc_sets, :force => true do |t|
44
+
45
+ create_table :dc_sets do |t|
41
46
  t.column :name, :string
42
47
  t.column :spec, :string
43
48
  t.column :description, :string
@@ -1,6 +1,5 @@
1
1
  require 'active_record'
2
2
  require 'oai'
3
- require "config/connection.rb"
4
3
 
5
4
  Dir.glob(File.dirname(__FILE__) + "/../models/*.rb").each do |lib|
6
5
  require lib
@@ -26,7 +25,7 @@ class CachingResumptionProvider < OAI::Provider::Base
26
25
  record_prefix 'oai:test'
27
26
  source_model ActiveRecordCachingWrapper.new(DCField, :limit => 25)
28
27
  end
29
-
28
+
30
29
 
31
30
  class ARLoader
32
31
  def self.load
@@ -37,7 +36,7 @@ class ARLoader
37
36
  DCField.create(fixtures[key])
38
37
  end
39
38
  end
40
-
39
+
41
40
  def self.unload
42
41
  DCField.delete_all
43
42
  end
@@ -1,31 +1,11 @@
1
1
  # Extend ActiveRecordModel to support sets
2
2
  class SetModel < OAI::Provider::ActiveRecordWrapper
3
-
3
+
4
4
  # Return all available sets
5
5
  def sets
6
- DCSet.find(:all)
6
+ DCSet.scoped
7
7
  end
8
8
 
9
- # Scope the find to a set relation if we get a set in the options
10
- def find(selector, opts={})
11
- if opts[:set]
12
- set = DCSet.find_by_spec(opts.delete(:set))
13
- conditions = sql_conditions(opts)
14
-
15
- if :all == selector
16
- set.dc_fields.find(selector, :conditions => conditions)
17
- else
18
- set.dc_fields.find(selector, :conditions => conditions)
19
- end
20
- else
21
- if :all == selector
22
- model.find(selector, :conditions => sql_conditions(opts))
23
- else
24
- model.find(selector, :conditions => sql_conditions(opts))
25
- end
26
- end
27
- end
28
-
29
9
  end
30
10
 
31
11
  class ARSetProvider < OAI::Provider::Base
@@ -33,4 +13,12 @@ class ARSetProvider < OAI::Provider::Base
33
13
  repository_url 'http://localhost'
34
14
  record_prefix = 'oai:test'
35
15
  source_model SetModel.new(DCField, :timestamp_field => 'date')
16
+ end
17
+
18
+ class ARExclusiveSetProvider < OAI::Provider::Base
19
+ repository_name 'ActiveRecord Set Based Provider'
20
+ repository_url 'http://localhost'
21
+ record_prefix = 'oai:test'
22
+ source_model OAI::Provider::ActiveRecordWrapper.new(
23
+ ExclusiveSetDCField, :timestamp_field => 'date')
36
24
  end
@@ -0,0 +1,34 @@
1
+ class TransactionalTestCase < Test::Unit::TestCase
2
+
3
+ def run(result, &block)
4
+ # Handle the default "you have no tests" test if it turns up
5
+ return if @method_name.to_s == "default_test"
6
+ ActiveRecord::Base.transaction do
7
+ load_fixtures
8
+ result = super(result, &block)
9
+ raise ActiveRecord::Rollback
10
+ end
11
+ result
12
+ end
13
+
14
+ protected
15
+
16
+ def load_fixtures
17
+ fixtures = YAML.load_file(
18
+ File.join(File.dirname(__FILE__), '..', 'fixtures', 'dc.yml')
19
+ )
20
+ disable_logging do
21
+ fixtures.keys.sort.each do |key|
22
+ DCField.create(fixtures[key])
23
+ end
24
+ end
25
+ end
26
+
27
+ def disable_logging
28
+ logger = ActiveRecord::Base.logger
29
+ ActiveRecord::Base.logger = nil
30
+ yield
31
+ ActiveRecord::Base.logger = logger
32
+ end
33
+
34
+ end
@@ -1,7 +1,7 @@
1
1
  class DCField < ActiveRecord::Base
2
- set_inheritance_column 'DONOTINHERIT'
3
- has_and_belongs_to_many :sets,
4
- :join_table => "dc_fields_dc_sets",
5
- :foreign_key => "dc_field_id",
2
+ inheritance_column = 'DONOTINHERIT'
3
+ has_and_belongs_to_many :sets,
4
+ :join_table => "dc_fields_dc_sets",
5
+ :foreign_key => "dc_field_id",
6
6
  :class_name => "DCSet"
7
7
  end
@@ -1,6 +1,7 @@
1
1
  class DCSet < ActiveRecord::Base
2
- has_and_belongs_to_many :dc_fields,
3
- :join_table => "dc_fields_dc_sets",
2
+ has_and_belongs_to_many :dc_fields,
3
+ :join_table => "dc_fields_dc_sets",
4
4
  :foreign_key => "dc_set_id",
5
5
  :class_name => "DCField"
6
+
6
7
  end
@@ -0,0 +1,11 @@
1
+ class ExclusiveSetDCField < ActiveRecord::Base
2
+ inheritance_column = 'DONOTINHERIT'
3
+
4
+ def self.sets
5
+ klass = Struct.new(:name, :spec)
6
+ self.uniq.pluck('`set`').compact.map do |spec|
7
+ klass.new("Set #{spec}", spec)
8
+ end
9
+ end
10
+
11
+ end
@@ -1,27 +1,30 @@
1
1
  require 'test_helper'
2
2
 
3
- class ActiveRecordProviderTest < Test::Unit::TestCase
3
+ class ActiveRecordProviderTest < TransactionalTestCase
4
4
 
5
5
  def test_identify
6
6
  assert @provider.identify =~ /ActiveRecord Based Provider/
7
7
  end
8
-
8
+
9
9
  def test_metadata_formats
10
10
  assert_nothing_raised { REXML::Document.new(@provider.list_metadata_formats) }
11
11
  doc = REXML::Document.new(@provider.list_metadata_formats)
12
12
  assert doc.elements['/OAI-PMH/ListMetadataFormats/metadataFormat/metadataPrefix'].text == 'oai_dc'
13
13
  end
14
-
14
+
15
15
  def test_metadata_formats_for_record
16
16
  record_id = DCField.find(:first).id
17
17
  assert_nothing_raised { REXML::Document.new(@provider.list_metadata_formats(:identifier => "oai:test/#{record_id}")) }
18
18
  doc = REXML::Document.new(@provider.list_metadata_formats)
19
19
  assert doc.elements['/OAI-PMH/ListMetadataFormats/metadataFormat/metadataPrefix'].text == 'oai_dc'
20
20
  end
21
-
21
+
22
22
  def test_list_records
23
- assert_nothing_raised { REXML::Document.new(@provider.list_records) }
24
- doc = REXML::Document.new(@provider.list_records)
23
+ assert_nothing_raised do
24
+ REXML::Document.new(@provider.list_records(:metadata_prefix => 'oai_dc'))
25
+ end
26
+ doc = REXML::Document.new(@provider.list_records(
27
+ :metadata_prefix => 'oai_dc'))
25
28
  assert_equal 100, doc.elements['OAI-PMH/ListRecords'].to_a.size
26
29
  end
27
30
 
@@ -33,52 +36,60 @@ class ActiveRecordProviderTest < Test::Unit::TestCase
33
36
 
34
37
  def test_get_record
35
38
  record_id = DCField.find(:first).id
36
- assert_nothing_raised { REXML::Document.new(@provider.get_record(:identifier => "oai:test/#{record_id}")) }
37
- doc = REXML::Document.new(@provider.get_record(:identifier => "#{record_id}"))
39
+ assert_nothing_raised do
40
+ REXML::Document.new(@provider.get_record(
41
+ :identifier => "oai:test/#{record_id}", :metadata_prefix => 'oai_dc'))
42
+ end
43
+ doc = REXML::Document.new(@provider.get_record(
44
+ :identifier => "#{record_id}", :metadata_prefix => 'oai_dc'))
38
45
  assert_equal "oai:test/#{record_id}", doc.elements['OAI-PMH/GetRecord/record/header/identifier'].text
39
46
  end
40
-
47
+
41
48
  def test_deleted
42
49
  record = DCField.find(:first)
43
50
  record.deleted = true;
44
51
  record.save
45
- doc = REXML::Document.new(@provider.get_record(:identifier => "oai:test/#{record.id}"))
52
+ doc = REXML::Document.new(@provider.get_record(
53
+ :identifier => "oai:test/#{record.id}", :metadata_prefix => 'oai_dc'))
46
54
  assert_equal "oai:test/#{record.id}", doc.elements['OAI-PMH/GetRecord/record/header/identifier'].text
47
55
  assert_equal 'deleted', doc.elements['OAI-PMH/GetRecord/record/header'].attributes["status"]
48
56
  end
49
-
57
+
50
58
  def test_from
51
59
  first_id = DCField.find(:first, :order => "id asc").id
52
60
  DCField.update_all(['updated_at = ?', Time.parse("January 1 2005")],
53
61
  "id < #{first_id + 90}")
54
62
  DCField.update_all(['updated_at = ?', Time.parse("June 1 2005")],
55
63
  "id < #{first_id + 10}")
56
-
64
+
57
65
  from_param = Time.parse("January 1 2006")
58
-
66
+
59
67
  doc = REXML::Document.new(
60
- @provider.list_records(:from => from_param)
61
- )
62
- assert_equal DCField.find(:all, :conditions => ["updated_at >= ?", from_param]).size,
68
+ @provider.list_records(
69
+ :metadata_prefix => 'oai_dc', :from => from_param)
70
+ )
71
+ assert_equal DCField.find(:all, :conditions => ["updated_at >= ?", from_param]).size,
63
72
  doc.elements['OAI-PMH/ListRecords'].size
64
73
 
65
74
  doc = REXML::Document.new(
66
- @provider.list_records(:from => Time.parse("May 30 2005"))
67
- )
75
+ @provider.list_records(
76
+ :metadata_prefix => 'oai_dc', :from => Time.parse("May 30 2005"))
77
+ )
68
78
  assert_equal 20, doc.elements['OAI-PMH/ListRecords'].to_a.size
69
79
  end
70
-
80
+
71
81
  def test_until
72
82
  first_id = DCField.find(:first, :order => "id asc").id
73
83
  DCField.update_all(['updated_at = ?', Time.parse("June 1 2005")],
74
84
  "id < #{first_id + 10}")
75
85
 
76
86
  doc = REXML::Document.new(
77
- @provider.list_records(:until => Time.parse("June 1 2005"))
78
- )
87
+ @provider.list_records(
88
+ :metadata_prefix => 'oai_dc', :until => Time.parse("June 1 2005"))
89
+ )
79
90
  assert_equal 10, doc.elements['OAI-PMH/ListRecords'].to_a.size
80
91
  end
81
-
92
+
82
93
  def test_from_and_until
83
94
  first_id = DCField.find(:first, :order => "id asc").id
84
95
  DCField.update_all(['updated_at = ?', Time.parse("June 1 2005")])
@@ -88,19 +99,47 @@ class ActiveRecordProviderTest < Test::Unit::TestCase
88
99
  "id < #{first_id + 10}")
89
100
 
90
101
  doc = REXML::Document.new(
91
- @provider.list_records(:from => Time.parse("June 3 2005"),
102
+ @provider.list_records(
103
+ :metadata_prefix => 'oai_dc',
104
+ :from => Time.parse("June 3 2005"),
92
105
  :until => Time.parse("June 16 2005"))
93
106
  )
94
107
  assert_equal 40, doc.elements['OAI-PMH/ListRecords'].to_a.size
95
108
  end
96
-
109
+
110
+ def test_handles_empty_collections
111
+ DCField.delete_all
112
+ assert DCField.count == 0
113
+ # Identify and ListMetadataFormats should return normally
114
+ test_identify
115
+ test_metadata_formats
116
+ # ListIdentifiers and ListRecords should return "noRecordsMatch" error code
117
+ assert_raises(OAI::NoMatchException) do
118
+ REXML::Document.new(@provider.list_identifiers)
119
+ end
120
+ assert_raises(OAI::NoMatchException) do
121
+ REXML::Document.new(@provider.list_records(:metadata_prefix => 'oai_dc'))
122
+ end
123
+ end
124
+
97
125
  def setup
98
126
  @provider = ARProvider.new
99
- ARLoader.load
100
127
  end
101
-
128
+
129
+ end
130
+
131
+ class ActiveRecordProviderTimezoneTest < ActiveRecordProviderTest
132
+
133
+ def setup
134
+ require 'active_record'
135
+ ActiveRecord::Base.default_timezone = :utc
136
+ super
137
+ end
138
+
102
139
  def teardown
103
- ARLoader.unload
140
+ require 'active_record'
141
+ ActiveRecord::Base.default_timezone = :local
142
+ super
104
143
  end
105
-
144
+
106
145
  end