mrt-ingest 0.0.4 → 0.0.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +5 -5
- data/.gitignore +5 -1
- data/.idea/.rakeTasks +7 -0
- data/.idea/encodings.xml +4 -0
- data/.idea/inspectionProfiles/Project_Default.xml +18 -0
- data/.idea/misc.xml +32 -0
- data/.idea/modules.xml +8 -0
- data/.idea/vcs.xml +6 -0
- data/.rubocop.yml +87 -0
- data/.ruby-version +2 -0
- data/CHANGES.md +25 -0
- data/Gemfile +1 -4
- data/README.md +56 -0
- data/Rakefile +36 -17
- data/lib/mrt/ingest.rb +1 -16
- data/lib/mrt/ingest/client.rb +11 -10
- data/lib/mrt/ingest/component.rb +78 -0
- data/lib/mrt/ingest/ingest_exception.rb +6 -0
- data/lib/mrt/ingest/iobject.rb +36 -93
- data/lib/mrt/ingest/message_digest.rb +8 -10
- data/lib/mrt/ingest/one_time_server.rb +34 -44
- data/lib/mrt/ingest/request.rb +67 -41
- data/lib/mrt/ingest/response.rb +6 -6
- data/mrt-ingest-ruby.iml +65 -0
- data/mrt-ingest.gemspec +22 -20
- data/spec/.rubocop.yml +16 -0
- data/spec/spec_helper.rb +36 -0
- data/spec/unit/data/file.txt +1 -0
- data/spec/unit/mrt/ingest/client_spec.rb +31 -0
- data/spec/unit/mrt/ingest/component_spec.rb +12 -0
- data/spec/unit/mrt/ingest/iobject_spec.rb +248 -0
- data/spec/unit/mrt/ingest/message_digest_spec.rb +38 -0
- data/spec/unit/mrt/ingest/one_time_server_spec.rb +113 -0
- data/spec/unit/mrt/ingest/request_spec.rb +25 -0
- data/spec/unit/mrt/ingest/response_spec.rb +56 -0
- metadata +95 -15
- data/README +0 -29
- data/test/test_client.rb +0 -40
- data/test/test_iobject.rb +0 -174
- data/test/test_request.rb +0 -37
- data/test/test_response.rb +0 -63
data/test/test_iobject.rb
DELETED
@@ -1,174 +0,0 @@
|
|
1
|
-
# Author:: Erik Hetzner (mailto:erik.hetzner@ucop.edu)
|
2
|
-
# Copyright:: Copyright (c) 2011, Regents of the University of California
|
3
|
-
|
4
|
-
require 'rubygems'
|
5
|
-
|
6
|
-
require 'test/unit'
|
7
|
-
require 'checkm'
|
8
|
-
require 'fakeweb'
|
9
|
-
require 'mocha'
|
10
|
-
require 'mrt/ingest'
|
11
|
-
require 'shoulda'
|
12
|
-
require 'open-uri'
|
13
|
-
require 'socket'
|
14
|
-
|
15
|
-
class TestIObject < Test::Unit::TestCase
|
16
|
-
def parse_object_manifest(iobject)
|
17
|
-
req = iobject.mk_request("profile", "submitter")
|
18
|
-
args = req.mk_args
|
19
|
-
return Checkm::Manifest.new(args['file'].read())
|
20
|
-
end
|
21
|
-
|
22
|
-
def write_to_tempfile(content)
|
23
|
-
tempfile = Tempfile.new('test_iobject')
|
24
|
-
tempfile << content
|
25
|
-
tempfile.open
|
26
|
-
return tempfile
|
27
|
-
end
|
28
|
-
|
29
|
-
def get_uri_for_name(iobject, name)
|
30
|
-
manifest = parse_object_manifest(iobject)
|
31
|
-
return manifest.entries.find { |entry|
|
32
|
-
entry.values[-2] == name
|
33
|
-
}
|
34
|
-
end
|
35
|
-
|
36
|
-
def parse_erc(erc)
|
37
|
-
arr = erc.lines.map do |line|
|
38
|
-
md = line.chomp.match(/^([^:]+):\s*(.*)$/)
|
39
|
-
[md[1], md[2]]
|
40
|
-
end.flatten
|
41
|
-
h = Hash[*arr]
|
42
|
-
h.delete("erc")
|
43
|
-
return h
|
44
|
-
end
|
45
|
-
|
46
|
-
def parse_erc_entry(erc_entry)
|
47
|
-
return parse_erc(open(erc_entry.values[0]).read())
|
48
|
-
end
|
49
|
-
|
50
|
-
def check_erc_content(iobject, asserted_erc)
|
51
|
-
erc_entry = get_uri_for_name(iobject, "mrt-erc.txt")
|
52
|
-
if erc_entry.nil?
|
53
|
-
assert(false, "Could not find mrt-erc.txt file!")
|
54
|
-
else
|
55
|
-
iobject.start_server()
|
56
|
-
assert_equal(asserted_erc, parse_erc_entry(erc_entry))
|
57
|
-
iobject.stop_server()
|
58
|
-
end
|
59
|
-
end
|
60
|
-
|
61
|
-
context "an iobject with local ID" do
|
62
|
-
setup do
|
63
|
-
@local_id = '10.1098/rstl.1665.0007'
|
64
|
-
@iobject = Mrt::Ingest::IObject.new(local_identifier: @local_id)
|
65
|
-
end
|
66
|
-
|
67
|
-
should "include that local ID in the resuest" do
|
68
|
-
request = @iobject.mk_request("profile", "submitter")
|
69
|
-
assert_equal(@local_id, request.local_identifier)
|
70
|
-
end
|
71
|
-
end
|
72
|
-
|
73
|
-
context "an iobject" do
|
74
|
-
setup do
|
75
|
-
@iobject = Mrt::Ingest::IObject.new
|
76
|
-
end
|
77
|
-
|
78
|
-
should "be able to add a URI component" do
|
79
|
-
@iobject.add_component(URI.parse("http://example.org/file"))
|
80
|
-
end
|
81
|
-
|
82
|
-
# TODO: remove prefetch code
|
83
|
-
# should "be able to add a URI component with prefetching, served locally" do
|
84
|
-
# @iobject.add_component(URI.parse("http://example.org/"), :prefetch=>true)
|
85
|
-
# manifest = parse_object_manifest(@iobject)
|
86
|
-
# manifest.entries.each do |entry|
|
87
|
-
# # check that all files are served locally
|
88
|
-
# uri = URI.parse(entry.values[0])
|
89
|
-
# assert_equal(Socket.gethostname, uri.host)
|
90
|
-
# end
|
91
|
-
# end
|
92
|
-
|
93
|
-
should "not be able to add a non-URI component" do
|
94
|
-
assert_raise(Mrt::Ingest::IngestException) do
|
95
|
-
@iobject.add_component("http://example.org/file")
|
96
|
-
end
|
97
|
-
end
|
98
|
-
|
99
|
-
should "be able to make a request" do
|
100
|
-
req = @iobject.mk_request("profile", "submitter")
|
101
|
-
assert_equal("profile", req.profile)
|
102
|
-
assert_equal("submitter", req.submitter)
|
103
|
-
end
|
104
|
-
end
|
105
|
-
|
106
|
-
context "the created request" do
|
107
|
-
setup do
|
108
|
-
@iobject = Mrt::Ingest::IObject.new
|
109
|
-
@manifest = parse_object_manifest(@iobject)
|
110
|
-
@erc_entry = get_uri_for_name(@iobject, "mrt-erc.txt")
|
111
|
-
end
|
112
|
-
|
113
|
-
should "generate a valid manifest file with more than one line" do
|
114
|
-
assert(@manifest.entries.length > 0, "Empty manifest?")
|
115
|
-
end
|
116
|
-
|
117
|
-
should "have a mrt-erc.txt entry, and it should be fetchable" do
|
118
|
-
if @erc_entry.nil?
|
119
|
-
assert(false, "Could not find mrt-erc.txt file!")
|
120
|
-
else
|
121
|
-
@iobject.start_server()
|
122
|
-
erc_lines = open(@erc_entry.values[0]).read().lines().to_a
|
123
|
-
@iobject.stop_server()
|
124
|
-
end
|
125
|
-
end
|
126
|
-
end
|
127
|
-
|
128
|
-
ERC_CONTENT = <<EOS
|
129
|
-
who: John Doe
|
130
|
-
what: Something
|
131
|
-
when: now
|
132
|
-
EOS
|
133
|
-
|
134
|
-
context "an iobject" do
|
135
|
-
should "be able to specify a file for ERC" do
|
136
|
-
erc_tempfile = write_to_tempfile(ERC_CONTENT)
|
137
|
-
iobject = Mrt::Ingest::IObject.new(:erc=>File.new(erc_tempfile.path))
|
138
|
-
check_erc_content(iobject, parse_erc(ERC_CONTENT))
|
139
|
-
end
|
140
|
-
|
141
|
-
should "be able to use a hash for ERC" do
|
142
|
-
erc = {
|
143
|
-
"who" => "John Doe",
|
144
|
-
"what" => "Something",
|
145
|
-
"when" => "now" }
|
146
|
-
iobject = Mrt::Ingest::IObject.new(:erc=>erc)
|
147
|
-
check_erc_content(iobject, erc)
|
148
|
-
end
|
149
|
-
end
|
150
|
-
|
151
|
-
FILE_CONTENT = <<EOS
|
152
|
-
Hello, world!
|
153
|
-
EOS
|
154
|
-
|
155
|
-
FILE_CONTENT_MD5 = "746308829575e17c3331bbcb00c0898b"
|
156
|
-
|
157
|
-
context "serving local files" do
|
158
|
-
should "be able to add a local file component" do
|
159
|
-
iobject = Mrt::Ingest::IObject.new
|
160
|
-
tempfile = write_to_tempfile(FILE_CONTENT)
|
161
|
-
iobject.add_component(tempfile, {:name => "helloworld" })
|
162
|
-
uri_entry = get_uri_for_name(iobject, "helloworld")
|
163
|
-
erc_entry = get_uri_for_name(iobject, "mrt-erc.txt")
|
164
|
-
manifest = parse_object_manifest(iobject)
|
165
|
-
if uri_entry.nil?
|
166
|
-
assert(false, "Could not find hosted file URI!")
|
167
|
-
else
|
168
|
-
iobject.start_server
|
169
|
-
assert_equal(FILE_CONTENT, open(uri_entry.values[0]).read())
|
170
|
-
iobject.stop_server()
|
171
|
-
end
|
172
|
-
end
|
173
|
-
end
|
174
|
-
end
|
data/test/test_request.rb
DELETED
@@ -1,37 +0,0 @@
|
|
1
|
-
# Author:: Erik Hetzner (mailto:erik.hetzner@ucop.edu)
|
2
|
-
# Copyright:: Copyright (c) 2011, Regents of the University of California
|
3
|
-
|
4
|
-
require 'rubygems'
|
5
|
-
|
6
|
-
require 'test/unit'
|
7
|
-
require 'fakeweb'
|
8
|
-
require 'mocha'
|
9
|
-
require 'mrt/ingest'
|
10
|
-
require 'shoulda'
|
11
|
-
|
12
|
-
class TestRequest < Test::Unit::TestCase
|
13
|
-
context "when creating a request" do
|
14
|
-
setup do
|
15
|
-
end
|
16
|
-
|
17
|
-
should "not supplying a required parameter should raise an exception" do
|
18
|
-
assert_raise(Mrt::Ingest::RequestException) do
|
19
|
-
Mrt::Ingest::Request.
|
20
|
-
new(:submitter => "jd/John Doe",
|
21
|
-
:type => "file")
|
22
|
-
end
|
23
|
-
|
24
|
-
assert_raise(Mrt::Ingest::RequestException) do
|
25
|
-
Mrt::Ingest::Request.
|
26
|
-
new(:profile => "demo_merritt",
|
27
|
-
:type => "file")
|
28
|
-
end
|
29
|
-
|
30
|
-
assert_raise(Mrt::Ingest::RequestException) do
|
31
|
-
Mrt::Ingest::Request.
|
32
|
-
new(:profile => "demo_merritt",
|
33
|
-
:submitter => "jd/John Doe")
|
34
|
-
end
|
35
|
-
end
|
36
|
-
end
|
37
|
-
end
|
data/test/test_response.rb
DELETED
@@ -1,63 +0,0 @@
|
|
1
|
-
# Author:: Erik Hetzner (mailto:erik.hetzner@ucop.edu)
|
2
|
-
# Copyright:: Copyright (c) 2011, Regents of the University of California
|
3
|
-
|
4
|
-
require 'rubygems'
|
5
|
-
|
6
|
-
require 'test/unit'
|
7
|
-
require 'fakeweb'
|
8
|
-
require 'mocha'
|
9
|
-
require 'mrt/ingest'
|
10
|
-
require 'shoulda'
|
11
|
-
|
12
|
-
class TestResponse < Test::Unit::TestCase
|
13
|
-
RESPONSE_JSON = <<EOS
|
14
|
-
{
|
15
|
-
|
16
|
-
"bat:batchState": {
|
17
|
-
"bat:batchID":"bid-8c0fa0c2-f3d7-4deb-bd49-f953f6752b59",
|
18
|
-
"bat:updateFlag":false,
|
19
|
-
"bat:targetQueue":"example.org:2181",
|
20
|
-
"bat:batchStatus":"QUEUED",
|
21
|
-
"bat:userAgent":"egh/Erik Hetzner",
|
22
|
-
"bat:submissionDate":"2011-08-31T15:40:26-07:00",
|
23
|
-
"bat:targetQueueNode":"/ingest.example.1",
|
24
|
-
"bat:batchProfile": {
|
25
|
-
"bat:owner":"ark:/99999/fk4tt4wsh",
|
26
|
-
"bat:creationDate":"2010-01-19T13:28:14-08:00",
|
27
|
-
"bat:targetStorage": {
|
28
|
-
"bat:storageLink":"http://example.org:35121",
|
29
|
-
"bat:nodeID":10
|
30
|
-
},
|
31
|
-
"bat:objectType":"MRT-curatorial",
|
32
|
-
"bat:modificationDate":"2010-01-26T23:28:14-08:00",
|
33
|
-
"bat:aggregateType":"",
|
34
|
-
"bat:objectMinterURL":"https://example.org/ezid/shoulder/ark:/99999/fk4",
|
35
|
-
"bat:collection": {
|
36
|
-
},
|
37
|
-
"bat:profileID":"merritt_content",
|
38
|
-
"bat:profileDescription":"Merritt demo content",
|
39
|
-
"bat:fixityURL":"http://example.org:33143",
|
40
|
-
"bat:contactsEmail": {
|
41
|
-
"bat:notification": {
|
42
|
-
"bat:contactEmail":"erik.hetzner@example.org"
|
43
|
-
}
|
44
|
-
},
|
45
|
-
"bat:identifierScheme":"ARK",
|
46
|
-
"bat:identifierNamespace":"99999",
|
47
|
-
"bat:objectRole":"MRT-content"
|
48
|
-
}
|
49
|
-
}
|
50
|
-
}
|
51
|
-
EOS
|
52
|
-
|
53
|
-
context "when creating a response" do
|
54
|
-
setup do
|
55
|
-
@response = Mrt::Ingest::Response.new(RESPONSE_JSON)
|
56
|
-
end
|
57
|
-
|
58
|
-
should "have the right properties" do
|
59
|
-
assert_equal("bid-8c0fa0c2-f3d7-4deb-bd49-f953f6752b59", @response.batch_id)
|
60
|
-
assert_equal(Time.at(1314830426), @response.submission_date)
|
61
|
-
end
|
62
|
-
end
|
63
|
-
end
|