oai 0.2.1 → 0.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (48) hide show
  1. data/README.md +28 -23
  2. data/Rakefile +14 -40
  3. data/examples/providers/dublin_core.rb +63 -63
  4. data/lib/oai/client.rb +131 -97
  5. data/lib/oai/client/list_identifiers.rb +1 -0
  6. data/lib/oai/client/list_records.rb +6 -5
  7. data/lib/oai/client/list_sets.rb +6 -5
  8. data/lib/oai/client/record.rb +6 -7
  9. data/lib/oai/client/response.rb +7 -4
  10. data/lib/oai/client/resumable.rb +42 -0
  11. data/lib/oai/harvester/shell.rb +40 -41
  12. data/lib/oai/provider.rb +85 -67
  13. data/lib/oai/provider/metadata_format/oai_dc.rb +5 -6
  14. data/lib/oai/provider/model/activerecord_caching_wrapper.rb +23 -25
  15. data/lib/oai/provider/model/activerecord_wrapper.rb +99 -51
  16. data/lib/oai/provider/response.rb +33 -31
  17. data/lib/oai/provider/response/get_record.rb +7 -7
  18. data/lib/oai/provider/response/list_records.rb +5 -4
  19. data/lib/oai/provider/response/record_response.rb +14 -14
  20. data/test/activerecord_provider/config/connection.rb +8 -4
  21. data/test/activerecord_provider/database/{ar_migration.rb → 0001_oaipmh_tables.rb} +17 -12
  22. data/test/activerecord_provider/helpers/providers.rb +2 -3
  23. data/test/activerecord_provider/helpers/set_provider.rb +10 -22
  24. data/test/activerecord_provider/helpers/transactional_test_case.rb +34 -0
  25. data/test/activerecord_provider/models/dc_field.rb +4 -4
  26. data/test/activerecord_provider/models/dc_set.rb +3 -2
  27. data/test/activerecord_provider/models/exclusive_set_dc_field.rb +11 -0
  28. data/test/activerecord_provider/tc_ar_provider.rb +67 -28
  29. data/test/activerecord_provider/tc_ar_sets_provider.rb +104 -18
  30. data/test/activerecord_provider/tc_caching_paging_provider.rb +6 -10
  31. data/test/activerecord_provider/tc_simple_paging_provider.rb +7 -11
  32. data/test/activerecord_provider/test_helper.rb +10 -0
  33. data/test/client/helpers/provider.rb +44 -47
  34. data/test/client/helpers/test_wrapper.rb +4 -16
  35. data/test/client/tc_http_client.rb +90 -2
  36. data/test/client/tc_list_identifiers.rb +22 -3
  37. data/test/client/tc_list_records.rb +17 -4
  38. data/test/client/tc_list_sets.rb +17 -2
  39. data/test/provider/models.rb +32 -30
  40. data/test/provider/tc_exceptions.rb +30 -20
  41. data/test/provider/tc_functional_tokens.rb +11 -6
  42. data/test/provider/tc_provider.rb +58 -24
  43. data/test/provider/tc_resumption_tokens.rb +6 -6
  44. data/test/provider/tc_simple_provider.rb +51 -26
  45. data/test/provider/test_helper.rb +7 -0
  46. metadata +67 -128
  47. data/test/activerecord_provider/config/database.yml +0 -6
  48. data/test/activerecord_provider/database/oaipmhtest +0 -0
@@ -1,8 +1,8 @@
1
1
  module OAI::Provider::Response
2
-
2
+
3
3
  class GetRecord < RecordResponse
4
- required_parameters :identifier
5
-
4
+ required_parameters :identifier, :metadata_prefix
5
+
6
6
  def to_xml
7
7
  id = extract_identifier(options.delete(:identifier))
8
8
  unless record = provider.model.find(id, options)
@@ -11,7 +11,7 @@ module OAI::Provider::Response
11
11
 
12
12
  response do |r|
13
13
  r.GetRecord do
14
- r.record do
14
+ r.record do
15
15
  header_for record
16
16
  data_for record unless deleted?(record)
17
17
  about_for record unless deleted?(record)
@@ -19,9 +19,9 @@ module OAI::Provider::Response
19
19
  end
20
20
  end
21
21
  end
22
-
22
+
23
23
  end
24
24
 
25
25
  end
26
-
27
-
26
+
27
+
@@ -1,7 +1,8 @@
1
1
  module OAI::Provider::Response
2
2
 
3
3
  class ListRecords < RecordResponse
4
-
4
+ required_parameters :metadata_prefix
5
+
5
6
  def to_xml
6
7
  result = provider.model.find(:all, options)
7
8
  # result may be an array of records, or a partial result
@@ -27,8 +28,8 @@ module OAI::Provider::Response
27
28
  end
28
29
  end
29
30
  end
30
-
31
+
31
32
  end
32
-
33
+
33
34
  end
34
-
35
+
@@ -2,16 +2,16 @@ module OAI::Provider::Response
2
2
  class RecordResponse < Base
3
3
  def self.inherited(klass)
4
4
  klass.valid_parameters :metadata_prefix, :from, :until, :set
5
- klass.default_parameters :metadata_prefix => "oai_dc",
5
+ klass.default_parameters :metadata_prefix => "oai_dc",
6
6
  :from => Proc.new {|x| Time.parse(x.provider.model.earliest.to_s) }, #-- OAI 2.0 hack - UTC
7
7
  :until => Proc.new {|x| Time.parse(x.provider.model.latest.to_s) } #-- OAI 2.0 hack - UTC
8
8
  end
9
-
9
+
10
10
  # emit record header
11
11
  def header_for(record)
12
12
  param = Hash.new
13
13
  param[:status] = 'deleted' if deleted?(record)
14
- @builder.header param do
14
+ @builder.header param do
15
15
  @builder.identifier identifier_for(record)
16
16
  @builder.datestamp timestamp_for(record)
17
17
  sets_for(record).each do |set|
@@ -33,7 +33,7 @@ module OAI::Provider::Response
33
33
  return unless provider.model.respond_to? :about
34
34
 
35
35
  about = provider.model.about(record)
36
- return if about.nil?
36
+ return if about.nil?
37
37
 
38
38
  unless about.is_a? Array
39
39
  about = [about]
@@ -42,43 +42,43 @@ module OAI::Provider::Response
42
42
  about.each do |a|
43
43
  @builder.about do
44
44
  @builder.target! << a
45
- end
45
+ end
46
46
  end
47
47
  end
48
-
48
+
49
49
  private
50
-
50
+
51
51
  def identifier_for(record)
52
52
  "#{provider.prefix}/#{record.id}"
53
53
  end
54
-
54
+
55
55
  def timestamp_for(record)
56
56
  record.send(provider.model.timestamp_field).utc.xmlschema
57
57
  end
58
-
58
+
59
59
  def sets_for(record)
60
60
  return [] unless record.respond_to?(:sets) and record.sets
61
61
  record.sets.respond_to?(:each) ? record.sets : [record.sets]
62
62
  end
63
-
63
+
64
64
  def requested_format
65
- format =
65
+ format =
66
66
  if options[:metadata_prefix]
67
67
  options[:metadata_prefix]
68
68
  elsif options[:resumption_token]
69
69
  OAI::Provider::ResumptionToken.extract_format(options[:resumption_token])
70
70
  end
71
71
  raise OAI::FormatException.new unless provider.format_supported?(format)
72
-
72
+
73
73
  format
74
74
  end
75
-
75
+
76
76
  def deleted?(record)
77
77
  return record.deleted? if record.respond_to?(:deleted?)
78
78
  return record.deleted if record.respond_to?(:deleted)
79
79
  return record.deleted_at if record.respond_to?(:deleted_at)
80
80
  false
81
81
  end
82
-
82
+
83
83
  end
84
84
  end
@@ -1,5 +1,9 @@
1
+ require 'active_record'
2
+ require 'logger'
3
+
1
4
  # Configure AR connection
2
- conn_info = YAML.load_file(
3
- File.join(File.dirname(__FILE__), "database.yml")
4
- )
5
- ActiveRecord::Base.establish_connection(conn_info)
5
+ #ActiveRecord::Base.logger = Logger.new(STDOUT)
6
+ ActiveRecord::Migration.verbose = false
7
+ ActiveRecord::Base.establish_connection :adapter => "sqlite3",
8
+ :database => ":memory:"
9
+ ActiveRecord::Migrator.up File.join(File.dirname(__FILE__), '..', 'database')
@@ -1,18 +1,16 @@
1
- ActiveRecord::Migration.verbose = false
2
-
3
- class OAIPMHTables < ActiveRecord::Migration
1
+ class OaipmhTables < ActiveRecord::Migration
4
2
  def self.up
5
- create_table :oai_tokens, :force => true do |t|
3
+ create_table :oai_tokens do |t|
6
4
  t.column :token, :string, :null => false
7
5
  t.column :created_at, :timestamp
8
6
  end
9
-
10
- create_table :oai_entries, :force => true do |t|
7
+
8
+ create_table :oai_entries do |t|
11
9
  t.column :record_id, :integer, :null => false
12
10
  t.column :oai_token_id, :integer, :null => false
13
11
  end
14
-
15
- create_table :dc_fields, :force => true do |t|
12
+
13
+ dc_fields = proc do |t|
16
14
  t.column :title, :string
17
15
  t.column :creator, :string
18
16
  t.column :subject, :string
@@ -31,13 +29,20 @@ class OAIPMHTables < ActiveRecord::Migration
31
29
  t.column :created_at, :datetime
32
30
  t.column :deleted, :boolean, :default => false
33
31
  end
34
-
35
- create_table :dc_fields_dc_sets, :force => true, :id => false do |t|
32
+
33
+ create_table :exclusive_set_dc_fields do |t|
34
+ dc_fields.call(t)
35
+ t.column :set, :string
36
+ end
37
+
38
+ create_table :dc_fields, &dc_fields
39
+
40
+ create_table :dc_fields_dc_sets, :id => false do |t|
36
41
  t.column :dc_field_id, :integer
37
42
  t.column :dc_set_id, :integer
38
43
  end
39
-
40
- create_table :dc_sets, :force => true do |t|
44
+
45
+ create_table :dc_sets do |t|
41
46
  t.column :name, :string
42
47
  t.column :spec, :string
43
48
  t.column :description, :string
@@ -1,6 +1,5 @@
1
1
  require 'active_record'
2
2
  require 'oai'
3
- require "config/connection.rb"
4
3
 
5
4
  Dir.glob(File.dirname(__FILE__) + "/../models/*.rb").each do |lib|
6
5
  require lib
@@ -26,7 +25,7 @@ class CachingResumptionProvider < OAI::Provider::Base
26
25
  record_prefix 'oai:test'
27
26
  source_model ActiveRecordCachingWrapper.new(DCField, :limit => 25)
28
27
  end
29
-
28
+
30
29
 
31
30
  class ARLoader
32
31
  def self.load
@@ -37,7 +36,7 @@ class ARLoader
37
36
  DCField.create(fixtures[key])
38
37
  end
39
38
  end
40
-
39
+
41
40
  def self.unload
42
41
  DCField.delete_all
43
42
  end
@@ -1,31 +1,11 @@
1
1
  # Extend ActiveRecordModel to support sets
2
2
  class SetModel < OAI::Provider::ActiveRecordWrapper
3
-
3
+
4
4
  # Return all available sets
5
5
  def sets
6
- DCSet.find(:all)
6
+ DCSet.scoped
7
7
  end
8
8
 
9
- # Scope the find to a set relation if we get a set in the options
10
- def find(selector, opts={})
11
- if opts[:set]
12
- set = DCSet.find_by_spec(opts.delete(:set))
13
- conditions = sql_conditions(opts)
14
-
15
- if :all == selector
16
- set.dc_fields.find(selector, :conditions => conditions)
17
- else
18
- set.dc_fields.find(selector, :conditions => conditions)
19
- end
20
- else
21
- if :all == selector
22
- model.find(selector, :conditions => sql_conditions(opts))
23
- else
24
- model.find(selector, :conditions => sql_conditions(opts))
25
- end
26
- end
27
- end
28
-
29
9
  end
30
10
 
31
11
  class ARSetProvider < OAI::Provider::Base
@@ -33,4 +13,12 @@ class ARSetProvider < OAI::Provider::Base
33
13
  repository_url 'http://localhost'
34
14
  record_prefix = 'oai:test'
35
15
  source_model SetModel.new(DCField, :timestamp_field => 'date')
16
+ end
17
+
18
+ class ARExclusiveSetProvider < OAI::Provider::Base
19
+ repository_name 'ActiveRecord Set Based Provider'
20
+ repository_url 'http://localhost'
21
+ record_prefix = 'oai:test'
22
+ source_model OAI::Provider::ActiveRecordWrapper.new(
23
+ ExclusiveSetDCField, :timestamp_field => 'date')
36
24
  end
@@ -0,0 +1,34 @@
1
+ class TransactionalTestCase < Test::Unit::TestCase
2
+
3
+ def run(result, &block)
4
+ # Handle the default "you have no tests" test if it turns up
5
+ return if @method_name.to_s == "default_test"
6
+ ActiveRecord::Base.transaction do
7
+ load_fixtures
8
+ result = super(result, &block)
9
+ raise ActiveRecord::Rollback
10
+ end
11
+ result
12
+ end
13
+
14
+ protected
15
+
16
+ def load_fixtures
17
+ fixtures = YAML.load_file(
18
+ File.join(File.dirname(__FILE__), '..', 'fixtures', 'dc.yml')
19
+ )
20
+ disable_logging do
21
+ fixtures.keys.sort.each do |key|
22
+ DCField.create(fixtures[key])
23
+ end
24
+ end
25
+ end
26
+
27
+ def disable_logging
28
+ logger = ActiveRecord::Base.logger
29
+ ActiveRecord::Base.logger = nil
30
+ yield
31
+ ActiveRecord::Base.logger = logger
32
+ end
33
+
34
+ end
@@ -1,7 +1,7 @@
1
1
  class DCField < ActiveRecord::Base
2
- set_inheritance_column 'DONOTINHERIT'
3
- has_and_belongs_to_many :sets,
4
- :join_table => "dc_fields_dc_sets",
5
- :foreign_key => "dc_field_id",
2
+ inheritance_column = 'DONOTINHERIT'
3
+ has_and_belongs_to_many :sets,
4
+ :join_table => "dc_fields_dc_sets",
5
+ :foreign_key => "dc_field_id",
6
6
  :class_name => "DCSet"
7
7
  end
@@ -1,6 +1,7 @@
1
1
  class DCSet < ActiveRecord::Base
2
- has_and_belongs_to_many :dc_fields,
3
- :join_table => "dc_fields_dc_sets",
2
+ has_and_belongs_to_many :dc_fields,
3
+ :join_table => "dc_fields_dc_sets",
4
4
  :foreign_key => "dc_set_id",
5
5
  :class_name => "DCField"
6
+
6
7
  end
@@ -0,0 +1,11 @@
1
+ class ExclusiveSetDCField < ActiveRecord::Base
2
+ inheritance_column = 'DONOTINHERIT'
3
+
4
+ def self.sets
5
+ klass = Struct.new(:name, :spec)
6
+ self.uniq.pluck('`set`').compact.map do |spec|
7
+ klass.new("Set #{spec}", spec)
8
+ end
9
+ end
10
+
11
+ end
@@ -1,27 +1,30 @@
1
1
  require 'test_helper'
2
2
 
3
- class ActiveRecordProviderTest < Test::Unit::TestCase
3
+ class ActiveRecordProviderTest < TransactionalTestCase
4
4
 
5
5
  def test_identify
6
6
  assert @provider.identify =~ /ActiveRecord Based Provider/
7
7
  end
8
-
8
+
9
9
  def test_metadata_formats
10
10
  assert_nothing_raised { REXML::Document.new(@provider.list_metadata_formats) }
11
11
  doc = REXML::Document.new(@provider.list_metadata_formats)
12
12
  assert doc.elements['/OAI-PMH/ListMetadataFormats/metadataFormat/metadataPrefix'].text == 'oai_dc'
13
13
  end
14
-
14
+
15
15
  def test_metadata_formats_for_record
16
16
  record_id = DCField.find(:first).id
17
17
  assert_nothing_raised { REXML::Document.new(@provider.list_metadata_formats(:identifier => "oai:test/#{record_id}")) }
18
18
  doc = REXML::Document.new(@provider.list_metadata_formats)
19
19
  assert doc.elements['/OAI-PMH/ListMetadataFormats/metadataFormat/metadataPrefix'].text == 'oai_dc'
20
20
  end
21
-
21
+
22
22
  def test_list_records
23
- assert_nothing_raised { REXML::Document.new(@provider.list_records) }
24
- doc = REXML::Document.new(@provider.list_records)
23
+ assert_nothing_raised do
24
+ REXML::Document.new(@provider.list_records(:metadata_prefix => 'oai_dc'))
25
+ end
26
+ doc = REXML::Document.new(@provider.list_records(
27
+ :metadata_prefix => 'oai_dc'))
25
28
  assert_equal 100, doc.elements['OAI-PMH/ListRecords'].to_a.size
26
29
  end
27
30
 
@@ -33,52 +36,60 @@ class ActiveRecordProviderTest < Test::Unit::TestCase
33
36
 
34
37
  def test_get_record
35
38
  record_id = DCField.find(:first).id
36
- assert_nothing_raised { REXML::Document.new(@provider.get_record(:identifier => "oai:test/#{record_id}")) }
37
- doc = REXML::Document.new(@provider.get_record(:identifier => "#{record_id}"))
39
+ assert_nothing_raised do
40
+ REXML::Document.new(@provider.get_record(
41
+ :identifier => "oai:test/#{record_id}", :metadata_prefix => 'oai_dc'))
42
+ end
43
+ doc = REXML::Document.new(@provider.get_record(
44
+ :identifier => "#{record_id}", :metadata_prefix => 'oai_dc'))
38
45
  assert_equal "oai:test/#{record_id}", doc.elements['OAI-PMH/GetRecord/record/header/identifier'].text
39
46
  end
40
-
47
+
41
48
  def test_deleted
42
49
  record = DCField.find(:first)
43
50
  record.deleted = true;
44
51
  record.save
45
- doc = REXML::Document.new(@provider.get_record(:identifier => "oai:test/#{record.id}"))
52
+ doc = REXML::Document.new(@provider.get_record(
53
+ :identifier => "oai:test/#{record.id}", :metadata_prefix => 'oai_dc'))
46
54
  assert_equal "oai:test/#{record.id}", doc.elements['OAI-PMH/GetRecord/record/header/identifier'].text
47
55
  assert_equal 'deleted', doc.elements['OAI-PMH/GetRecord/record/header'].attributes["status"]
48
56
  end
49
-
57
+
50
58
  def test_from
51
59
  first_id = DCField.find(:first, :order => "id asc").id
52
60
  DCField.update_all(['updated_at = ?', Time.parse("January 1 2005")],
53
61
  "id < #{first_id + 90}")
54
62
  DCField.update_all(['updated_at = ?', Time.parse("June 1 2005")],
55
63
  "id < #{first_id + 10}")
56
-
64
+
57
65
  from_param = Time.parse("January 1 2006")
58
-
66
+
59
67
  doc = REXML::Document.new(
60
- @provider.list_records(:from => from_param)
61
- )
62
- assert_equal DCField.find(:all, :conditions => ["updated_at >= ?", from_param]).size,
68
+ @provider.list_records(
69
+ :metadata_prefix => 'oai_dc', :from => from_param)
70
+ )
71
+ assert_equal DCField.find(:all, :conditions => ["updated_at >= ?", from_param]).size,
63
72
  doc.elements['OAI-PMH/ListRecords'].size
64
73
 
65
74
  doc = REXML::Document.new(
66
- @provider.list_records(:from => Time.parse("May 30 2005"))
67
- )
75
+ @provider.list_records(
76
+ :metadata_prefix => 'oai_dc', :from => Time.parse("May 30 2005"))
77
+ )
68
78
  assert_equal 20, doc.elements['OAI-PMH/ListRecords'].to_a.size
69
79
  end
70
-
80
+
71
81
  def test_until
72
82
  first_id = DCField.find(:first, :order => "id asc").id
73
83
  DCField.update_all(['updated_at = ?', Time.parse("June 1 2005")],
74
84
  "id < #{first_id + 10}")
75
85
 
76
86
  doc = REXML::Document.new(
77
- @provider.list_records(:until => Time.parse("June 1 2005"))
78
- )
87
+ @provider.list_records(
88
+ :metadata_prefix => 'oai_dc', :until => Time.parse("June 1 2005"))
89
+ )
79
90
  assert_equal 10, doc.elements['OAI-PMH/ListRecords'].to_a.size
80
91
  end
81
-
92
+
82
93
  def test_from_and_until
83
94
  first_id = DCField.find(:first, :order => "id asc").id
84
95
  DCField.update_all(['updated_at = ?', Time.parse("June 1 2005")])
@@ -88,19 +99,47 @@ class ActiveRecordProviderTest < Test::Unit::TestCase
88
99
  "id < #{first_id + 10}")
89
100
 
90
101
  doc = REXML::Document.new(
91
- @provider.list_records(:from => Time.parse("June 3 2005"),
102
+ @provider.list_records(
103
+ :metadata_prefix => 'oai_dc',
104
+ :from => Time.parse("June 3 2005"),
92
105
  :until => Time.parse("June 16 2005"))
93
106
  )
94
107
  assert_equal 40, doc.elements['OAI-PMH/ListRecords'].to_a.size
95
108
  end
96
-
109
+
110
+ def test_handles_empty_collections
111
+ DCField.delete_all
112
+ assert DCField.count == 0
113
+ # Identify and ListMetadataFormats should return normally
114
+ test_identify
115
+ test_metadata_formats
116
+ # ListIdentifiers and ListRecords should return "noRecordsMatch" error code
117
+ assert_raises(OAI::NoMatchException) do
118
+ REXML::Document.new(@provider.list_identifiers)
119
+ end
120
+ assert_raises(OAI::NoMatchException) do
121
+ REXML::Document.new(@provider.list_records(:metadata_prefix => 'oai_dc'))
122
+ end
123
+ end
124
+
97
125
  def setup
98
126
  @provider = ARProvider.new
99
- ARLoader.load
100
127
  end
101
-
128
+
129
+ end
130
+
131
+ class ActiveRecordProviderTimezoneTest < ActiveRecordProviderTest
132
+
133
+ def setup
134
+ require 'active_record'
135
+ ActiveRecord::Base.default_timezone = :utc
136
+ super
137
+ end
138
+
102
139
  def teardown
103
- ARLoader.unload
140
+ require 'active_record'
141
+ ActiveRecord::Base.default_timezone = :local
142
+ super
104
143
  end
105
-
144
+
106
145
  end