mrt-ingest 0.0.2 → 0.0.3

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 7d30e43c2e0b21873ca263108300477a4a2926fe2ab7edb72d3327900253e1ac
+  data.tar.gz: 73466ed8f6640a0756d13fcbe4c79f9a8f98f7db8518a2a9fa047bb77bc576ed
+SHA512:
+  metadata.gz: 85ef3dfa325d3f0bbdc705faa630b1c0e0bacd9436586ff753c74bd9d18db254b8c86c9bb363804b0b9853c259e513eab81f66065b17194ef4f18be9aed55c93
+  data.tar.gz: 6b0d2cf3d02703ccc7e715093dff00f8934a8499d519c8c0d64a039b0a57c230f284a8f57001cc06935715e1c9f1ea72a32b0b0034ab13b6efc1452dfcd2552f

data/Gemfile

@@ -6,5 +6,6 @@ gemspec
 group :test do
   gem "fakeweb"
   gem "mocha"
-  gem "checkm", ">=0.0.6"
+  gem "checkm", "0.0.6"
+  gem "shoulda"
 end

data/lib/mrt/ingest/iobject.rb

@@ -4,6 +4,8 @@
 require 'mrt/ingest'
 require 'tempfile'
 require 'uri'
+require 'open-uri'
+require 'digest/md5'
 
 module Mrt
   module Ingest
@@ -15,7 +17,9 @@ module Mrt
         @digest = options[:digest]
         @mime_type = options[:mime_type]
         @size = options[:size]
-        
+        # @prefetch = options[:prefetch] || false
+        @prefetch = false
+
         case where
         when File, Tempfile
           @name = File.basename(where.path) if @name.nil?
@@ -26,7 +30,20 @@ module Mrt
           @size = File.size(where.path) if @size.nil?
         when URI
           @name = File.basename(where.to_s) if @name.nil?
-          @uri = where
+          if @prefetch then
+            digest = Digest::MD5.new()
+            @uri, ignore = server.add_file do |f|
+              open(where, (options[:prefetch_options] || {})) do |u|
+                while (buff = u.read(1024)) do
+                  f << buff
+                  digest << buff
+                end
+              end
+            end
+            @digest = Mrt::Ingest::MessageDigest::MD5.new(digest.hexdigest)
+          else
+            @uri = where
+          end
         else
           raise IngestException.new("Trying to add a component that is not a File or URI")
         end
@@ -75,6 +92,7 @@ module Mrt
                           Component.new(@server, @erc, :name => 'mrt-erc.txt')
                         when Hash
                           uri_str, path = @server.add_file do |f|
+                            f.write("erc:\n")
                             @erc.each_pair do |k, v|
                               f.write("#{k}: #{v}\n")
                             end
@@ -96,6 +114,7 @@ module Mrt
               :type               => "object-manifest",
               :submitter          => submitter,
               :profile            => profile,
+              :local_identifier => @local_identifier,
               :primary_identifier => @primary_identifier)
       end
 

data/lib/mrt/ingest/one_time_server.rb

@@ -9,7 +9,7 @@ module Mrt
     class OneTimeServer
       # Find an open port, starting with start and adding one until we get
       # an open port
-      def get_open_port(start=8080)
+      def get_open_port(start=8081)
         try_port = start
         while (true)
           begin
@@ -83,8 +83,13 @@ module Mrt
       end
       
       def start_server
-        @thread = Thread.new do
-          @server.start
+        if @thread.nil? then
+          @thread = Thread.new do
+            @server.start
+          end
+        end
+        while (@server.status != :Running) do
+          sleep(0.1)
         end
         return @thread
       end

data/lib/mrt/ingest/response.rb

@@ -10,19 +10,19 @@ module Mrt
   module Ingest
     class Response
       def initialize(data)
-        @parsed = JSON.parse(data)['batchState']
+        @parsed = JSON.parse(data)['bat:batchState']
       end
       
       def batch_id
-        return @parsed['batchID']
+        return @parsed['bat:batchID']
       end
       
       def user_agent
-        return @parsed['userAgent']
+        return @parsed['bat:userAgent']
       end
       
       def submission_date
-        return Time.parse(@parsed['submissionDate'])
+        return Time.parse(@parsed['bat:submissionDate'])
       end
     end
   end

data/mrt-ingest.gemspec

@@ -1,24 +1,22 @@
 # -*- encoding: utf-8 -*-
 $:.push File.expand_path("../lib", __FILE__)
-require "mrt/ingest/version"
 
 Gem::Specification.new do |s|
   s.name        = "mrt-ingest"
-  s.version     = "0.0.2"
+  s.version     = "0.0.3"
   s.platform    = Gem::Platform::RUBY
-  s.authors     = ["Erik Hetzner"]
-  s.email       = ["erik.hetzner@ucop.edu"]
-  s.homepage    = "http://bitbucket.org/merritt/mrt-ingest-ruby"
+  s.authors     = ["Mark Reyes", "David Moles"]
+  s.email       = ["mark.reyes@ucop.edu", "david.moles@ucop.edu"]
+  s.homepage    = "https://github.com/CDLUC3/mrt-ingest-ruby"
   s.summary     = %q{A client for Merritt ingest.}
-  s.description = %q{A client for the Merritt ingest system. More details available from http://wiki.ucop.edu/display/curation.}
+  s.description = %q{A client for the Merritt ingest system. More details available from https://github.com/CDLUC3/mrt-doc/wiki}
+  s.license     = "BSD-3-Clause"
 
-  s.add_dependency "json", ">=1.5.0"
-  s.add_dependency "rest-client", ">=1.6.0"
+  s.add_dependency "json", "~> 1.5", ">=1.5.0"
+  s.add_dependency "rest-client", "~> 1.6", ">=1.6.0"
 
-  s.rubyforge_project = "mrt-ingest"
-
-  s.files         = `hg locate`.split("\n")
-  s.test_files    = `hg locate --include '{spec,features}'`.split("\n")
-  s.executables   = `hg locate --include bin`.split("\n").map{ |f| File.basename(f) }
+  s.files         = `git ls-files -z`.split("\x0")
+  s.test_files    = s.files.grep(%r{^(test|spec|features)/})
+  s.executables   = s.files.grep(%r{^bin/}) { |f| File.basename(f) }
   s.require_paths = ["lib"]
 end

data/test/test_iobject.rb

@@ -9,6 +9,7 @@ require 'mocha'
 require 'mrt/ingest'
 require 'shoulda'
 require 'open-uri'
+require 'socket'
 
 class TestIObject < Test::Unit::TestCase
   def parse_object_manifest(iobject)
@@ -32,7 +33,13 @@ class TestIObject < Test::Unit::TestCase
   end
 
   def parse_erc(erc)
-    return Hash[erc.map { |l| l.chomp.split(/:\s+/) }]
+    arr = erc.lines.map do |line|
+      md = line.chomp.match(/^([^:]+):\s*(.*)$/)
+      [md[1], md[2]]
+    end.flatten
+    h = Hash[*arr]
+    h.delete("erc")
+    return h
   end
 
   def parse_erc_entry(erc_entry)
@@ -59,6 +66,16 @@ class TestIObject < Test::Unit::TestCase
       @iobject.add_component(URI.parse("http://example.org/file"))
     end
 
+    should "be able to add a URI component with prefetching, served locally" do
+      @iobject.add_component(URI.parse("http://example.org/"), :prefetch=>true)
+      manifest = parse_object_manifest(@iobject)
+      manifest.entries.each do |entry|
+        # check that all files are served locally
+        uri = URI.parse(entry.values[0])
+        assert_equal(Socket.gethostname, uri.host)
+      end
+    end
+
     should "not be able to add a non-URI component" do
       assert_raise(Mrt::Ingest::IngestException) do
         @iobject.add_component("http://example.org/file")

metadata

@@ -1,65 +1,65 @@
---- !ruby/object:Gem::Specification 
+--- !ruby/object:Gem::Specification
 name: mrt-ingest
-version: !ruby/object:Gem::Version 
-  hash: 27
-  prerelease: 
-  segments: 
-  - 0
-  - 0
-  - 2
-  version: 0.0.2
+version: !ruby/object:Gem::Version
+  version: 0.0.3
 platform: ruby
-authors: 
-- Erik Hetzner
+authors:
+- Mark Reyes
+- David Moles
 autorequire: 
 bindir: bin
 cert_chain: []
-
-date: 2011-12-20 00:00:00 Z
-dependencies: 
-- !ruby/object:Gem::Dependency 
+date: 2018-10-23 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
   name: json
-  prerelease: false
-  requirement: &id001 !ruby/object:Gem::Requirement 
-    none: false
-    requirements: 
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.5'
     - - ">="
-      - !ruby/object:Gem::Version 
-        hash: 3
-        segments: 
-        - 1
-        - 5
-        - 0
+      - !ruby/object:Gem::Version
         version: 1.5.0
   type: :runtime
-  version_requirements: *id001
-- !ruby/object:Gem::Dependency 
-  name: rest-client
   prerelease: false
-  requirement: &id002 !ruby/object:Gem::Requirement 
-    none: false
-    requirements: 
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.5'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.5.0
+- !ruby/object:Gem::Dependency
+  name: rest-client
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.6'
     - - ">="
-      - !ruby/object:Gem::Version 
-        hash: 15
-        segments: 
-        - 1
-        - 6
-        - 0
+      - !ruby/object:Gem::Version
         version: 1.6.0
   type: :runtime
-  version_requirements: *id002
-description: A client for the Merritt ingest system. More details available from http://wiki.ucop.edu/display/curation.
-email: 
-- erik.hetzner@ucop.edu
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.6'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.6.0
+description: A client for the Merritt ingest system. More details available from https://github.com/CDLUC3/mrt-doc/wiki
+email:
+- mark.reyes@ucop.edu
+- david.moles@ucop.edu
 executables: []
-
 extensions: []
-
 extra_rdoc_files: []
-
-files: 
-- .hgignore
+files:
+- ".hgignore"
 - Gemfile
 - LICENSE
 - README
@@ -76,38 +76,32 @@ files:
 - test/test_iobject.rb
 - test/test_request.rb
 - test/test_response.rb
-homepage: http://bitbucket.org/merritt/mrt-ingest-ruby
-licenses: []
-
+homepage: https://github.com/CDLUC3/mrt-ingest-ruby
+licenses:
+- BSD-3-Clause
+metadata: {}
 post_install_message: 
 rdoc_options: []
-
-require_paths: 
+require_paths:
 - lib
-required_ruby_version: !ruby/object:Gem::Requirement 
-  none: false
-  requirements: 
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
   - - ">="
-    - !ruby/object:Gem::Version 
-      hash: 3
-      segments: 
-      - 0
-      version: "0"
-required_rubygems_version: !ruby/object:Gem::Requirement 
-  none: false
-  requirements: 
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
   - - ">="
-    - !ruby/object:Gem::Version 
-      hash: 3
-      segments: 
-      - 0
-      version: "0"
+    - !ruby/object:Gem::Version
+      version: '0'
 requirements: []
-
-rubyforge_project: mrt-ingest
-rubygems_version: 1.8.12
+rubyforge_project: 
+rubygems_version: 2.7.7
 signing_key: 
-specification_version: 3
+specification_version: 4
 summary: A client for Merritt ingest.
-test_files: []
-
+test_files:
+- test/test_client.rb
+- test/test_iobject.rb
+- test/test_request.rb
+- test/test_response.rb