mrt-ingest 0.0.4 → 0.0.5
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +5 -5
- data/.gitignore +5 -1
- data/.idea/.rakeTasks +7 -0
- data/.idea/encodings.xml +4 -0
- data/.idea/inspectionProfiles/Project_Default.xml +18 -0
- data/.idea/misc.xml +32 -0
- data/.idea/modules.xml +8 -0
- data/.idea/vcs.xml +6 -0
- data/.rubocop.yml +87 -0
- data/.ruby-version +2 -0
- data/CHANGES.md +25 -0
- data/Gemfile +1 -4
- data/README.md +56 -0
- data/Rakefile +36 -17
- data/lib/mrt/ingest.rb +1 -16
- data/lib/mrt/ingest/client.rb +11 -10
- data/lib/mrt/ingest/component.rb +78 -0
- data/lib/mrt/ingest/ingest_exception.rb +6 -0
- data/lib/mrt/ingest/iobject.rb +36 -93
- data/lib/mrt/ingest/message_digest.rb +8 -10
- data/lib/mrt/ingest/one_time_server.rb +34 -44
- data/lib/mrt/ingest/request.rb +67 -41
- data/lib/mrt/ingest/response.rb +6 -6
- data/mrt-ingest-ruby.iml +65 -0
- data/mrt-ingest.gemspec +22 -20
- data/spec/.rubocop.yml +16 -0
- data/spec/spec_helper.rb +36 -0
- data/spec/unit/data/file.txt +1 -0
- data/spec/unit/mrt/ingest/client_spec.rb +31 -0
- data/spec/unit/mrt/ingest/component_spec.rb +12 -0
- data/spec/unit/mrt/ingest/iobject_spec.rb +248 -0
- data/spec/unit/mrt/ingest/message_digest_spec.rb +38 -0
- data/spec/unit/mrt/ingest/one_time_server_spec.rb +113 -0
- data/spec/unit/mrt/ingest/request_spec.rb +25 -0
- data/spec/unit/mrt/ingest/response_spec.rb +56 -0
- metadata +95 -15
- data/README +0 -29
- data/test/test_client.rb +0 -40
- data/test/test_iobject.rb +0 -174
- data/test/test_request.rb +0 -37
- data/test/test_response.rb +0 -63
data/test/test_iobject.rb
DELETED
@@ -1,174 +0,0 @@
|
|
1
|
-
# Author:: Erik Hetzner (mailto:erik.hetzner@ucop.edu)
|
2
|
-
# Copyright:: Copyright (c) 2011, Regents of the University of California
|
3
|
-
|
4
|
-
require 'rubygems'
|
5
|
-
|
6
|
-
require 'test/unit'
|
7
|
-
require 'checkm'
|
8
|
-
require 'fakeweb'
|
9
|
-
require 'mocha'
|
10
|
-
require 'mrt/ingest'
|
11
|
-
require 'shoulda'
|
12
|
-
require 'open-uri'
|
13
|
-
require 'socket'
|
14
|
-
|
15
|
-
class TestIObject < Test::Unit::TestCase
|
16
|
-
def parse_object_manifest(iobject)
|
17
|
-
req = iobject.mk_request("profile", "submitter")
|
18
|
-
args = req.mk_args
|
19
|
-
return Checkm::Manifest.new(args['file'].read())
|
20
|
-
end
|
21
|
-
|
22
|
-
def write_to_tempfile(content)
|
23
|
-
tempfile = Tempfile.new('test_iobject')
|
24
|
-
tempfile << content
|
25
|
-
tempfile.open
|
26
|
-
return tempfile
|
27
|
-
end
|
28
|
-
|
29
|
-
def get_uri_for_name(iobject, name)
|
30
|
-
manifest = parse_object_manifest(iobject)
|
31
|
-
return manifest.entries.find { |entry|
|
32
|
-
entry.values[-2] == name
|
33
|
-
}
|
34
|
-
end
|
35
|
-
|
36
|
-
def parse_erc(erc)
|
37
|
-
arr = erc.lines.map do |line|
|
38
|
-
md = line.chomp.match(/^([^:]+):\s*(.*)$/)
|
39
|
-
[md[1], md[2]]
|
40
|
-
end.flatten
|
41
|
-
h = Hash[*arr]
|
42
|
-
h.delete("erc")
|
43
|
-
return h
|
44
|
-
end
|
45
|
-
|
46
|
-
def parse_erc_entry(erc_entry)
|
47
|
-
return parse_erc(open(erc_entry.values[0]).read())
|
48
|
-
end
|
49
|
-
|
50
|
-
def check_erc_content(iobject, asserted_erc)
|
51
|
-
erc_entry = get_uri_for_name(iobject, "mrt-erc.txt")
|
52
|
-
if erc_entry.nil?
|
53
|
-
assert(false, "Could not find mrt-erc.txt file!")
|
54
|
-
else
|
55
|
-
iobject.start_server()
|
56
|
-
assert_equal(asserted_erc, parse_erc_entry(erc_entry))
|
57
|
-
iobject.stop_server()
|
58
|
-
end
|
59
|
-
end
|
60
|
-
|
61
|
-
context "an iobject with local ID" do
|
62
|
-
setup do
|
63
|
-
@local_id = '10.1098/rstl.1665.0007'
|
64
|
-
@iobject = Mrt::Ingest::IObject.new(local_identifier: @local_id)
|
65
|
-
end
|
66
|
-
|
67
|
-
should "include that local ID in the resuest" do
|
68
|
-
request = @iobject.mk_request("profile", "submitter")
|
69
|
-
assert_equal(@local_id, request.local_identifier)
|
70
|
-
end
|
71
|
-
end
|
72
|
-
|
73
|
-
context "an iobject" do
|
74
|
-
setup do
|
75
|
-
@iobject = Mrt::Ingest::IObject.new
|
76
|
-
end
|
77
|
-
|
78
|
-
should "be able to add a URI component" do
|
79
|
-
@iobject.add_component(URI.parse("http://example.org/file"))
|
80
|
-
end
|
81
|
-
|
82
|
-
# TODO: remove prefetch code
|
83
|
-
# should "be able to add a URI component with prefetching, served locally" do
|
84
|
-
# @iobject.add_component(URI.parse("http://example.org/"), :prefetch=>true)
|
85
|
-
# manifest = parse_object_manifest(@iobject)
|
86
|
-
# manifest.entries.each do |entry|
|
87
|
-
# # check that all files are served locally
|
88
|
-
# uri = URI.parse(entry.values[0])
|
89
|
-
# assert_equal(Socket.gethostname, uri.host)
|
90
|
-
# end
|
91
|
-
# end
|
92
|
-
|
93
|
-
should "not be able to add a non-URI component" do
|
94
|
-
assert_raise(Mrt::Ingest::IngestException) do
|
95
|
-
@iobject.add_component("http://example.org/file")
|
96
|
-
end
|
97
|
-
end
|
98
|
-
|
99
|
-
should "be able to make a request" do
|
100
|
-
req = @iobject.mk_request("profile", "submitter")
|
101
|
-
assert_equal("profile", req.profile)
|
102
|
-
assert_equal("submitter", req.submitter)
|
103
|
-
end
|
104
|
-
end
|
105
|
-
|
106
|
-
context "the created request" do
|
107
|
-
setup do
|
108
|
-
@iobject = Mrt::Ingest::IObject.new
|
109
|
-
@manifest = parse_object_manifest(@iobject)
|
110
|
-
@erc_entry = get_uri_for_name(@iobject, "mrt-erc.txt")
|
111
|
-
end
|
112
|
-
|
113
|
-
should "generate a valid manifest file with more than one line" do
|
114
|
-
assert(@manifest.entries.length > 0, "Empty manifest?")
|
115
|
-
end
|
116
|
-
|
117
|
-
should "have a mrt-erc.txt entry, and it should be fetchable" do
|
118
|
-
if @erc_entry.nil?
|
119
|
-
assert(false, "Could not find mrt-erc.txt file!")
|
120
|
-
else
|
121
|
-
@iobject.start_server()
|
122
|
-
erc_lines = open(@erc_entry.values[0]).read().lines().to_a
|
123
|
-
@iobject.stop_server()
|
124
|
-
end
|
125
|
-
end
|
126
|
-
end
|
127
|
-
|
128
|
-
ERC_CONTENT = <<EOS
|
129
|
-
who: John Doe
|
130
|
-
what: Something
|
131
|
-
when: now
|
132
|
-
EOS
|
133
|
-
|
134
|
-
context "an iobject" do
|
135
|
-
should "be able to specify a file for ERC" do
|
136
|
-
erc_tempfile = write_to_tempfile(ERC_CONTENT)
|
137
|
-
iobject = Mrt::Ingest::IObject.new(:erc=>File.new(erc_tempfile.path))
|
138
|
-
check_erc_content(iobject, parse_erc(ERC_CONTENT))
|
139
|
-
end
|
140
|
-
|
141
|
-
should "be able to use a hash for ERC" do
|
142
|
-
erc = {
|
143
|
-
"who" => "John Doe",
|
144
|
-
"what" => "Something",
|
145
|
-
"when" => "now" }
|
146
|
-
iobject = Mrt::Ingest::IObject.new(:erc=>erc)
|
147
|
-
check_erc_content(iobject, erc)
|
148
|
-
end
|
149
|
-
end
|
150
|
-
|
151
|
-
FILE_CONTENT = <<EOS
|
152
|
-
Hello, world!
|
153
|
-
EOS
|
154
|
-
|
155
|
-
FILE_CONTENT_MD5 = "746308829575e17c3331bbcb00c0898b"
|
156
|
-
|
157
|
-
context "serving local files" do
|
158
|
-
should "be able to add a local file component" do
|
159
|
-
iobject = Mrt::Ingest::IObject.new
|
160
|
-
tempfile = write_to_tempfile(FILE_CONTENT)
|
161
|
-
iobject.add_component(tempfile, {:name => "helloworld" })
|
162
|
-
uri_entry = get_uri_for_name(iobject, "helloworld")
|
163
|
-
erc_entry = get_uri_for_name(iobject, "mrt-erc.txt")
|
164
|
-
manifest = parse_object_manifest(iobject)
|
165
|
-
if uri_entry.nil?
|
166
|
-
assert(false, "Could not find hosted file URI!")
|
167
|
-
else
|
168
|
-
iobject.start_server
|
169
|
-
assert_equal(FILE_CONTENT, open(uri_entry.values[0]).read())
|
170
|
-
iobject.stop_server()
|
171
|
-
end
|
172
|
-
end
|
173
|
-
end
|
174
|
-
end
|
data/test/test_request.rb
DELETED
@@ -1,37 +0,0 @@
|
|
1
|
-
# Author:: Erik Hetzner (mailto:erik.hetzner@ucop.edu)
|
2
|
-
# Copyright:: Copyright (c) 2011, Regents of the University of California
|
3
|
-
|
4
|
-
require 'rubygems'
|
5
|
-
|
6
|
-
require 'test/unit'
|
7
|
-
require 'fakeweb'
|
8
|
-
require 'mocha'
|
9
|
-
require 'mrt/ingest'
|
10
|
-
require 'shoulda'
|
11
|
-
|
12
|
-
class TestRequest < Test::Unit::TestCase
|
13
|
-
context "when creating a request" do
|
14
|
-
setup do
|
15
|
-
end
|
16
|
-
|
17
|
-
should "not supplying a required parameter should raise an exception" do
|
18
|
-
assert_raise(Mrt::Ingest::RequestException) do
|
19
|
-
Mrt::Ingest::Request.
|
20
|
-
new(:submitter => "jd/John Doe",
|
21
|
-
:type => "file")
|
22
|
-
end
|
23
|
-
|
24
|
-
assert_raise(Mrt::Ingest::RequestException) do
|
25
|
-
Mrt::Ingest::Request.
|
26
|
-
new(:profile => "demo_merritt",
|
27
|
-
:type => "file")
|
28
|
-
end
|
29
|
-
|
30
|
-
assert_raise(Mrt::Ingest::RequestException) do
|
31
|
-
Mrt::Ingest::Request.
|
32
|
-
new(:profile => "demo_merritt",
|
33
|
-
:submitter => "jd/John Doe")
|
34
|
-
end
|
35
|
-
end
|
36
|
-
end
|
37
|
-
end
|
data/test/test_response.rb
DELETED
@@ -1,63 +0,0 @@
|
|
1
|
-
# Author:: Erik Hetzner (mailto:erik.hetzner@ucop.edu)
|
2
|
-
# Copyright:: Copyright (c) 2011, Regents of the University of California
|
3
|
-
|
4
|
-
require 'rubygems'
|
5
|
-
|
6
|
-
require 'test/unit'
|
7
|
-
require 'fakeweb'
|
8
|
-
require 'mocha'
|
9
|
-
require 'mrt/ingest'
|
10
|
-
require 'shoulda'
|
11
|
-
|
12
|
-
class TestResponse < Test::Unit::TestCase
|
13
|
-
RESPONSE_JSON = <<EOS
|
14
|
-
{
|
15
|
-
|
16
|
-
"bat:batchState": {
|
17
|
-
"bat:batchID":"bid-8c0fa0c2-f3d7-4deb-bd49-f953f6752b59",
|
18
|
-
"bat:updateFlag":false,
|
19
|
-
"bat:targetQueue":"example.org:2181",
|
20
|
-
"bat:batchStatus":"QUEUED",
|
21
|
-
"bat:userAgent":"egh/Erik Hetzner",
|
22
|
-
"bat:submissionDate":"2011-08-31T15:40:26-07:00",
|
23
|
-
"bat:targetQueueNode":"/ingest.example.1",
|
24
|
-
"bat:batchProfile": {
|
25
|
-
"bat:owner":"ark:/99999/fk4tt4wsh",
|
26
|
-
"bat:creationDate":"2010-01-19T13:28:14-08:00",
|
27
|
-
"bat:targetStorage": {
|
28
|
-
"bat:storageLink":"http://example.org:35121",
|
29
|
-
"bat:nodeID":10
|
30
|
-
},
|
31
|
-
"bat:objectType":"MRT-curatorial",
|
32
|
-
"bat:modificationDate":"2010-01-26T23:28:14-08:00",
|
33
|
-
"bat:aggregateType":"",
|
34
|
-
"bat:objectMinterURL":"https://example.org/ezid/shoulder/ark:/99999/fk4",
|
35
|
-
"bat:collection": {
|
36
|
-
},
|
37
|
-
"bat:profileID":"merritt_content",
|
38
|
-
"bat:profileDescription":"Merritt demo content",
|
39
|
-
"bat:fixityURL":"http://example.org:33143",
|
40
|
-
"bat:contactsEmail": {
|
41
|
-
"bat:notification": {
|
42
|
-
"bat:contactEmail":"erik.hetzner@example.org"
|
43
|
-
}
|
44
|
-
},
|
45
|
-
"bat:identifierScheme":"ARK",
|
46
|
-
"bat:identifierNamespace":"99999",
|
47
|
-
"bat:objectRole":"MRT-content"
|
48
|
-
}
|
49
|
-
}
|
50
|
-
}
|
51
|
-
EOS
|
52
|
-
|
53
|
-
context "when creating a response" do
|
54
|
-
setup do
|
55
|
-
@response = Mrt::Ingest::Response.new(RESPONSE_JSON)
|
56
|
-
end
|
57
|
-
|
58
|
-
should "have the right properties" do
|
59
|
-
assert_equal("bid-8c0fa0c2-f3d7-4deb-bd49-f953f6752b59", @response.batch_id)
|
60
|
-
assert_equal(Time.at(1314830426), @response.submission_date)
|
61
|
-
end
|
62
|
-
end
|
63
|
-
end
|