mrt-ingest 0.0.2 → 0.0.3

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml ADDED
@@ -0,0 +1,7 @@
1
+ ---
2
+ SHA256:
3
+ metadata.gz: 7d30e43c2e0b21873ca263108300477a4a2926fe2ab7edb72d3327900253e1ac
4
+ data.tar.gz: 73466ed8f6640a0756d13fcbe4c79f9a8f98f7db8518a2a9fa047bb77bc576ed
5
+ SHA512:
6
+ metadata.gz: 85ef3dfa325d3f0bbdc705faa630b1c0e0bacd9436586ff753c74bd9d18db254b8c86c9bb363804b0b9853c259e513eab81f66065b17194ef4f18be9aed55c93
7
+ data.tar.gz: 6b0d2cf3d02703ccc7e715093dff00f8934a8499d519c8c0d64a039b0a57c230f284a8f57001cc06935715e1c9f1ea72a32b0b0034ab13b6efc1452dfcd2552f
data/Gemfile CHANGED
@@ -6,5 +6,6 @@ gemspec
6
6
  group :test do
7
7
  gem "fakeweb"
8
8
  gem "mocha"
9
- gem "checkm", ">=0.0.6"
9
+ gem "checkm", "0.0.6"
10
+ gem "shoulda"
10
11
  end
@@ -4,6 +4,8 @@
4
4
  require 'mrt/ingest'
5
5
  require 'tempfile'
6
6
  require 'uri'
7
+ require 'open-uri'
8
+ require 'digest/md5'
7
9
 
8
10
  module Mrt
9
11
  module Ingest
@@ -15,7 +17,9 @@ module Mrt
15
17
  @digest = options[:digest]
16
18
  @mime_type = options[:mime_type]
17
19
  @size = options[:size]
18
-
20
+ # @prefetch = options[:prefetch] || false
21
+ @prefetch = false
22
+
19
23
  case where
20
24
  when File, Tempfile
21
25
  @name = File.basename(where.path) if @name.nil?
@@ -26,7 +30,20 @@ module Mrt
26
30
  @size = File.size(where.path) if @size.nil?
27
31
  when URI
28
32
  @name = File.basename(where.to_s) if @name.nil?
29
- @uri = where
33
+ if @prefetch then
34
+ digest = Digest::MD5.new()
35
+ @uri, ignore = server.add_file do |f|
36
+ open(where, (options[:prefetch_options] || {})) do |u|
37
+ while (buff = u.read(1024)) do
38
+ f << buff
39
+ digest << buff
40
+ end
41
+ end
42
+ end
43
+ @digest = Mrt::Ingest::MessageDigest::MD5.new(digest.hexdigest)
44
+ else
45
+ @uri = where
46
+ end
30
47
  else
31
48
  raise IngestException.new("Trying to add a component that is not a File or URI")
32
49
  end
@@ -75,6 +92,7 @@ module Mrt
75
92
  Component.new(@server, @erc, :name => 'mrt-erc.txt')
76
93
  when Hash
77
94
  uri_str, path = @server.add_file do |f|
95
+ f.write("erc:\n")
78
96
  @erc.each_pair do |k, v|
79
97
  f.write("#{k}: #{v}\n")
80
98
  end
@@ -96,6 +114,7 @@ module Mrt
96
114
  :type => "object-manifest",
97
115
  :submitter => submitter,
98
116
  :profile => profile,
117
+ :local_identifier => @local_identifier,
99
118
  :primary_identifier => @primary_identifier)
100
119
  end
101
120
 
@@ -9,7 +9,7 @@ module Mrt
9
9
  class OneTimeServer
10
10
  # Find an open port, starting with start and adding one until we get
11
11
  # an open port
12
- def get_open_port(start=8080)
12
+ def get_open_port(start=8081)
13
13
  try_port = start
14
14
  while (true)
15
15
  begin
@@ -83,8 +83,13 @@ module Mrt
83
83
  end
84
84
 
85
85
  def start_server
86
- @thread = Thread.new do
87
- @server.start
86
+ if @thread.nil? then
87
+ @thread = Thread.new do
88
+ @server.start
89
+ end
90
+ end
91
+ while (@server.status != :Running) do
92
+ sleep(0.1)
88
93
  end
89
94
  return @thread
90
95
  end
@@ -10,19 +10,19 @@ module Mrt
10
10
  module Ingest
11
11
  class Response
12
12
  def initialize(data)
13
- @parsed = JSON.parse(data)['batchState']
13
+ @parsed = JSON.parse(data)['bat:batchState']
14
14
  end
15
15
 
16
16
  def batch_id
17
- return @parsed['batchID']
17
+ return @parsed['bat:batchID']
18
18
  end
19
19
 
20
20
  def user_agent
21
- return @parsed['userAgent']
21
+ return @parsed['bat:userAgent']
22
22
  end
23
23
 
24
24
  def submission_date
25
- return Time.parse(@parsed['submissionDate'])
25
+ return Time.parse(@parsed['bat:submissionDate'])
26
26
  end
27
27
  end
28
28
  end
data/mrt-ingest.gemspec CHANGED
@@ -1,24 +1,22 @@
1
1
  # -*- encoding: utf-8 -*-
2
2
  $:.push File.expand_path("../lib", __FILE__)
3
- require "mrt/ingest/version"
4
3
 
5
4
  Gem::Specification.new do |s|
6
5
  s.name = "mrt-ingest"
7
- s.version = "0.0.2"
6
+ s.version = "0.0.3"
8
7
  s.platform = Gem::Platform::RUBY
9
- s.authors = ["Erik Hetzner"]
10
- s.email = ["erik.hetzner@ucop.edu"]
11
- s.homepage = "http://bitbucket.org/merritt/mrt-ingest-ruby"
8
+ s.authors = ["Mark Reyes", "David Moles"]
9
+ s.email = ["mark.reyes@ucop.edu", "david.moles@ucop.edu"]
10
+ s.homepage = "https://github.com/CDLUC3/mrt-ingest-ruby"
12
11
  s.summary = %q{A client for Merritt ingest.}
13
- s.description = %q{A client for the Merritt ingest system. More details available from http://wiki.ucop.edu/display/curation.}
12
+ s.description = %q{A client for the Merritt ingest system. More details available from https://github.com/CDLUC3/mrt-doc/wiki}
13
+ s.license = "BSD-3-Clause"
14
14
 
15
- s.add_dependency "json", ">=1.5.0"
16
- s.add_dependency "rest-client", ">=1.6.0"
15
+ s.add_dependency "json", "~> 1.5", ">=1.5.0"
16
+ s.add_dependency "rest-client", "~> 1.6", ">=1.6.0"
17
17
 
18
- s.rubyforge_project = "mrt-ingest"
19
-
20
- s.files = `hg locate`.split("\n")
21
- s.test_files = `hg locate --include '{spec,features}'`.split("\n")
22
- s.executables = `hg locate --include bin`.split("\n").map{ |f| File.basename(f) }
18
+ s.files = `git ls-files -z`.split("\x0")
19
+ s.test_files = s.files.grep(%r{^(test|spec|features)/})
20
+ s.executables = s.files.grep(%r{^bin/}) { |f| File.basename(f) }
23
21
  s.require_paths = ["lib"]
24
22
  end
data/test/test_iobject.rb CHANGED
@@ -9,6 +9,7 @@ require 'mocha'
9
9
  require 'mrt/ingest'
10
10
  require 'shoulda'
11
11
  require 'open-uri'
12
+ require 'socket'
12
13
 
13
14
  class TestIObject < Test::Unit::TestCase
14
15
  def parse_object_manifest(iobject)
@@ -32,7 +33,13 @@ class TestIObject < Test::Unit::TestCase
32
33
  end
33
34
 
34
35
  def parse_erc(erc)
35
- return Hash[erc.map { |l| l.chomp.split(/:\s+/) }]
36
+ arr = erc.lines.map do |line|
37
+ md = line.chomp.match(/^([^:]+):\s*(.*)$/)
38
+ [md[1], md[2]]
39
+ end.flatten
40
+ h = Hash[*arr]
41
+ h.delete("erc")
42
+ return h
36
43
  end
37
44
 
38
45
  def parse_erc_entry(erc_entry)
@@ -59,6 +66,16 @@ class TestIObject < Test::Unit::TestCase
59
66
  @iobject.add_component(URI.parse("http://example.org/file"))
60
67
  end
61
68
 
69
+ should "be able to add a URI component with prefetching, served locally" do
70
+ @iobject.add_component(URI.parse("http://example.org/"), :prefetch=>true)
71
+ manifest = parse_object_manifest(@iobject)
72
+ manifest.entries.each do |entry|
73
+ # check that all files are served locally
74
+ uri = URI.parse(entry.values[0])
75
+ assert_equal(Socket.gethostname, uri.host)
76
+ end
77
+ end
78
+
62
79
  should "not be able to add a non-URI component" do
63
80
  assert_raise(Mrt::Ingest::IngestException) do
64
81
  @iobject.add_component("http://example.org/file")
metadata CHANGED
@@ -1,65 +1,65 @@
1
- --- !ruby/object:Gem::Specification
1
+ --- !ruby/object:Gem::Specification
2
2
  name: mrt-ingest
3
- version: !ruby/object:Gem::Version
4
- hash: 27
5
- prerelease:
6
- segments:
7
- - 0
8
- - 0
9
- - 2
10
- version: 0.0.2
3
+ version: !ruby/object:Gem::Version
4
+ version: 0.0.3
11
5
  platform: ruby
12
- authors:
13
- - Erik Hetzner
6
+ authors:
7
+ - Mark Reyes
8
+ - David Moles
14
9
  autorequire:
15
10
  bindir: bin
16
11
  cert_chain: []
17
-
18
- date: 2011-12-20 00:00:00 Z
19
- dependencies:
20
- - !ruby/object:Gem::Dependency
12
+ date: 2018-10-23 00:00:00.000000000 Z
13
+ dependencies:
14
+ - !ruby/object:Gem::Dependency
21
15
  name: json
22
- prerelease: false
23
- requirement: &id001 !ruby/object:Gem::Requirement
24
- none: false
25
- requirements:
16
+ requirement: !ruby/object:Gem::Requirement
17
+ requirements:
18
+ - - "~>"
19
+ - !ruby/object:Gem::Version
20
+ version: '1.5'
26
21
  - - ">="
27
- - !ruby/object:Gem::Version
28
- hash: 3
29
- segments:
30
- - 1
31
- - 5
32
- - 0
22
+ - !ruby/object:Gem::Version
33
23
  version: 1.5.0
34
24
  type: :runtime
35
- version_requirements: *id001
36
- - !ruby/object:Gem::Dependency
37
- name: rest-client
38
25
  prerelease: false
39
- requirement: &id002 !ruby/object:Gem::Requirement
40
- none: false
41
- requirements:
26
+ version_requirements: !ruby/object:Gem::Requirement
27
+ requirements:
28
+ - - "~>"
29
+ - !ruby/object:Gem::Version
30
+ version: '1.5'
31
+ - - ">="
32
+ - !ruby/object:Gem::Version
33
+ version: 1.5.0
34
+ - !ruby/object:Gem::Dependency
35
+ name: rest-client
36
+ requirement: !ruby/object:Gem::Requirement
37
+ requirements:
38
+ - - "~>"
39
+ - !ruby/object:Gem::Version
40
+ version: '1.6'
42
41
  - - ">="
43
- - !ruby/object:Gem::Version
44
- hash: 15
45
- segments:
46
- - 1
47
- - 6
48
- - 0
42
+ - !ruby/object:Gem::Version
49
43
  version: 1.6.0
50
44
  type: :runtime
51
- version_requirements: *id002
52
- description: A client for the Merritt ingest system. More details available from http://wiki.ucop.edu/display/curation.
53
- email:
54
- - erik.hetzner@ucop.edu
45
+ prerelease: false
46
+ version_requirements: !ruby/object:Gem::Requirement
47
+ requirements:
48
+ - - "~>"
49
+ - !ruby/object:Gem::Version
50
+ version: '1.6'
51
+ - - ">="
52
+ - !ruby/object:Gem::Version
53
+ version: 1.6.0
54
+ description: A client for the Merritt ingest system. More details available from https://github.com/CDLUC3/mrt-doc/wiki
55
+ email:
56
+ - mark.reyes@ucop.edu
57
+ - david.moles@ucop.edu
55
58
  executables: []
56
-
57
59
  extensions: []
58
-
59
60
  extra_rdoc_files: []
60
-
61
- files:
62
- - .hgignore
61
+ files:
62
+ - ".hgignore"
63
63
  - Gemfile
64
64
  - LICENSE
65
65
  - README
@@ -76,38 +76,32 @@ files:
76
76
  - test/test_iobject.rb
77
77
  - test/test_request.rb
78
78
  - test/test_response.rb
79
- homepage: http://bitbucket.org/merritt/mrt-ingest-ruby
80
- licenses: []
81
-
79
+ homepage: https://github.com/CDLUC3/mrt-ingest-ruby
80
+ licenses:
81
+ - BSD-3-Clause
82
+ metadata: {}
82
83
  post_install_message:
83
84
  rdoc_options: []
84
-
85
- require_paths:
85
+ require_paths:
86
86
  - lib
87
- required_ruby_version: !ruby/object:Gem::Requirement
88
- none: false
89
- requirements:
87
+ required_ruby_version: !ruby/object:Gem::Requirement
88
+ requirements:
90
89
  - - ">="
91
- - !ruby/object:Gem::Version
92
- hash: 3
93
- segments:
94
- - 0
95
- version: "0"
96
- required_rubygems_version: !ruby/object:Gem::Requirement
97
- none: false
98
- requirements:
90
+ - !ruby/object:Gem::Version
91
+ version: '0'
92
+ required_rubygems_version: !ruby/object:Gem::Requirement
93
+ requirements:
99
94
  - - ">="
100
- - !ruby/object:Gem::Version
101
- hash: 3
102
- segments:
103
- - 0
104
- version: "0"
95
+ - !ruby/object:Gem::Version
96
+ version: '0'
105
97
  requirements: []
106
-
107
- rubyforge_project: mrt-ingest
108
- rubygems_version: 1.8.12
98
+ rubyforge_project:
99
+ rubygems_version: 2.7.7
109
100
  signing_key:
110
- specification_version: 3
101
+ specification_version: 4
111
102
  summary: A client for Merritt ingest.
112
- test_files: []
113
-
103
+ test_files:
104
+ - test/test_client.rb
105
+ - test/test_iobject.rb
106
+ - test/test_request.rb
107
+ - test/test_response.rb