mrt-ingest 0.0.4 → 0.0.5

data/lib/mrt/ingest/response.rb

@@ -12,17 +12,17 @@ module Mrt
       def initialize(data)
         @parsed = JSON.parse(data)['bat:batchState']
       end
-      
+
       def batch_id
-        return @parsed['bat:batchID']
+        @parsed['bat:batchID']
       end
-      
+
       def user_agent
-        return @parsed['bat:userAgent']
+        @parsed['bat:userAgent']
       end
-      
+
       def submission_date
-        return Time.parse(@parsed['bat:submissionDate'])
+        Time.parse(@parsed['bat:submissionDate'])
       end
     end
   end

data/mrt-ingest-ruby.iml

@@ -0,0 +1,65 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<module type="RUBY_MODULE" version="4">
+  <component name="NewModuleRootManager" inherit-compiler-output="true">
+    <exclude-output />
+    <content url="file://$MODULE_DIR$">
+      <sourceFolder url="file://$MODULE_DIR$/lib" isTestSource="false" />
+      <sourceFolder url="file://$MODULE_DIR$/spec" isTestSource="true" />
+    </content>
+    <orderEntry type="jdk" jdkName="RVM: ruby-2.4.4" jdkType="RUBY_SDK" />
+    <orderEntry type="sourceFolder" forTests="false" />
+    <orderEntry type="library" scope="PROVIDED" name="activesupport (v5.2.3, RVM: ruby-2.4.4) [gem]" level="application" />
+    <orderEntry type="library" scope="PROVIDED" name="addressable (v2.6.0, RVM: ruby-2.4.4) [gem]" level="application" />
+    <orderEntry type="library" scope="PROVIDED" name="ansi (v1.5.0, RVM: ruby-2.4.4) [gem]" level="application" />
+    <orderEntry type="library" scope="PROVIDED" name="ast (v2.4.0, RVM: ruby-2.4.4) [gem]" level="application" />
+    <orderEntry type="library" scope="PROVIDED" name="bundler (v1.17.1, RVM: ruby-2.4.4) [gem]" level="application" />
+    <orderEntry type="library" scope="PROVIDED" name="checkm (v0.0.6, RVM: ruby-2.4.4) [gem]" level="application" />
+    <orderEntry type="library" scope="PROVIDED" name="concurrent-ruby (v1.1.5, RVM: ruby-2.4.4) [gem]" level="application" />
+    <orderEntry type="library" scope="PROVIDED" name="crack (v0.4.3, RVM: ruby-2.4.4) [gem]" level="application" />
+    <orderEntry type="library" scope="PROVIDED" name="diff-lcs (v1.3, RVM: ruby-2.4.4) [gem]" level="application" />
+    <orderEntry type="library" scope="PROVIDED" name="docile (v1.3.1, RVM: ruby-2.4.4) [gem]" level="application" />
+    <orderEntry type="library" scope="PROVIDED" name="domain_name (v0.5.20180417, RVM: ruby-2.4.4) [gem]" level="application" />
+    <orderEntry type="library" scope="PROVIDED" name="hashdiff (v0.3.9, RVM: ruby-2.4.4) [gem]" level="application" />
+    <orderEntry type="library" scope="PROVIDED" name="hirb (v0.7.3, RVM: ruby-2.4.4) [gem]" level="application" />
+    <orderEntry type="library" scope="PROVIDED" name="http-cookie (v1.0.3, RVM: ruby-2.4.4) [gem]" level="application" />
+    <orderEntry type="library" scope="PROVIDED" name="i18n (v1.6.0, RVM: ruby-2.4.4) [gem]" level="application" />
+    <orderEntry type="library" scope="PROVIDED" name="jaro_winkler (v1.5.2, RVM: ruby-2.4.4) [gem]" level="application" />
+    <orderEntry type="library" scope="PROVIDED" name="json (v2.2.0, RVM: ruby-2.4.4) [gem]" level="application" />
+    <orderEntry type="library" scope="PROVIDED" name="metaclass (v0.0.4, RVM: ruby-2.4.4) [gem]" level="application" />
+    <orderEntry type="library" scope="PROVIDED" name="mime-types (v3.2.2, RVM: ruby-2.4.4) [gem]" level="application" />
+    <orderEntry type="library" scope="PROVIDED" name="mime-types-data (v3.2019.0331, RVM: ruby-2.4.4) [gem]" level="application" />
+    <orderEntry type="library" scope="PROVIDED" name="minitest (v5.11.3, RVM: ruby-2.4.4) [gem]" level="application" />
+    <orderEntry type="library" scope="PROVIDED" name="mocha (v1.8.0, RVM: ruby-2.4.4) [gem]" level="application" />
+    <orderEntry type="library" scope="PROVIDED" name="netrc (v0.11.0, RVM: ruby-2.4.4) [gem]" level="application" />
+    <orderEntry type="library" scope="PROVIDED" name="parallel (v1.17.0, RVM: ruby-2.4.4) [gem]" level="application" />
+    <orderEntry type="library" scope="PROVIDED" name="parser (v2.6.3.0, RVM: ruby-2.4.4) [gem]" level="application" />
+    <orderEntry type="library" scope="PROVIDED" name="public_suffix (v3.0.3, RVM: ruby-2.4.4) [gem]" level="application" />
+    <orderEntry type="library" scope="PROVIDED" name="rainbow (v3.0.0, RVM: ruby-2.4.4) [gem]" level="application" />
+    <orderEntry type="library" scope="PROVIDED" name="rake (v12.3.2, RVM: ruby-2.4.4) [gem]" level="application" />
+    <orderEntry type="library" scope="PROVIDED" name="rest-client (v2.0.2, RVM: ruby-2.4.4) [gem]" level="application" />
+    <orderEntry type="library" scope="PROVIDED" name="rspec (v3.8.0, RVM: ruby-2.4.4) [gem]" level="application" />
+    <orderEntry type="library" scope="PROVIDED" name="rspec-core (v3.8.0, RVM: ruby-2.4.4) [gem]" level="application" />
+    <orderEntry type="library" scope="PROVIDED" name="rspec-expectations (v3.8.3, RVM: ruby-2.4.4) [gem]" level="application" />
+    <orderEntry type="library" scope="PROVIDED" name="rspec-mocks (v3.8.0, RVM: ruby-2.4.4) [gem]" level="application" />
+    <orderEntry type="library" scope="PROVIDED" name="rspec-support (v3.8.0, RVM: ruby-2.4.4) [gem]" level="application" />
+    <orderEntry type="library" scope="PROVIDED" name="rubocop (v0.69.0, RVM: ruby-2.4.4) [gem]" level="application" />
+    <orderEntry type="library" scope="PROVIDED" name="ruby-progressbar (v1.10.0, RVM: ruby-2.4.4) [gem]" level="application" />
+    <orderEntry type="library" scope="PROVIDED" name="safe_yaml (v1.0.5, RVM: ruby-2.4.4) [gem]" level="application" />
+    <orderEntry type="library" scope="PROVIDED" name="shoulda (v3.6.0, RVM: ruby-2.4.4) [gem]" level="application" />
+    <orderEntry type="library" scope="PROVIDED" name="shoulda-context (v1.2.2, RVM: ruby-2.4.4) [gem]" level="application" />
+    <orderEntry type="library" scope="PROVIDED" name="shoulda-matchers (v3.1.3, RVM: ruby-2.4.4) [gem]" level="application" />
+    <orderEntry type="library" scope="PROVIDED" name="simplecov (v0.16.1, RVM: ruby-2.4.4) [gem]" level="application" />
+    <orderEntry type="library" scope="PROVIDED" name="simplecov-console (v0.4.2, RVM: ruby-2.4.4) [gem]" level="application" />
+    <orderEntry type="library" scope="PROVIDED" name="simplecov-html (v0.10.2, RVM: ruby-2.4.4) [gem]" level="application" />
+    <orderEntry type="library" scope="PROVIDED" name="thread_safe (v0.3.6, RVM: ruby-2.4.4) [gem]" level="application" />
+    <orderEntry type="library" scope="PROVIDED" name="tzinfo (v1.2.5, RVM: ruby-2.4.4) [gem]" level="application" />
+    <orderEntry type="library" scope="PROVIDED" name="unf (v0.1.4, RVM: ruby-2.4.4) [gem]" level="application" />
+    <orderEntry type="library" scope="PROVIDED" name="unf_ext (v0.0.7.6, RVM: ruby-2.4.4) [gem]" level="application" />
+    <orderEntry type="library" scope="PROVIDED" name="unicode-display_width (v1.6.0, RVM: ruby-2.4.4) [gem]" level="application" />
+    <orderEntry type="library" scope="PROVIDED" name="webmock (v3.5.1, RVM: ruby-2.4.4) [gem]" level="application" />
+  </component>
+  <component name="RModuleSettingsStorage">
+    <LOAD_PATH number="2" string0="$MODULE_DIR$/lib" string1="$MODULE_DIR$/spec" />
+    <I18N_FOLDERS number="0" />
+  </component>
+</module>

data/mrt-ingest.gemspec

@@ -1,30 +1,32 @@
-$:.push File.expand_path("../lib", __FILE__)
+$LOAD_PATH.push File.expand_path('lib', __dir__)
 
 Gem::Specification.new do |s|
-  s.name        = "mrt-ingest"
-  s.version     = "0.0.4"
+  s.name        = 'mrt-ingest'
+  s.version     = '0.0.5'
   s.platform    = Gem::Platform::RUBY
-  s.authors     = ["Mark Reyes", "David Moles"]
-  s.email       = ["mark.reyes@ucop.edu", "david.moles@ucop.edu"]
-  s.homepage    = "https://github.com/CDLUC3/mrt-ingest-ruby"
-  s.summary     = %q{A client for Merritt ingest.}
-  s.description = %q{A client for the Merritt ingest system. More details available from https://github.com/CDLUC3/mrt-doc/wiki}
-  s.license     = "BSD-3-Clause"
+  s.authors     = ['Mark Reyes', 'David Moles']
+  s.email       = ['mark.reyes@ucop.edu', 'david.moles@ucop.edu']
+  s.homepage    = 'https://github.com/CDLUC3/mrt-ingest-ruby'
+  s.summary     = 'A client for Merritt ingest.'
+  s.description = 'A client for the Merritt ingest system. More details available from https://github.com/CDLUC3/mrt-doc/wiki'
+  s.license     = 'BSD-3-Clause'
 
-  s.add_dependency "json", "~> 2.0"
-  s.add_dependency "rest-client", "~> 2.0"
+  s.add_dependency 'json', '~> 2.0'
+  s.add_dependency 'rest-client', '~> 2.0'
 
-  s.add_development_dependency "bundler", '~> 1.16'
-  s.add_development_dependency "checkm", "0.0.6"
-  # TODO: put this back in once there's a 2.4-compatible release
-  # s.add_development_dependency "fakeweb"
-  s.add_development_dependency "mocha", '~> 1.7'
-  s.add_development_dependency "rake", '~> 12.0'
-  s.add_development_dependency "shoulda", '~> 3.6'
-  s.add_development_dependency "test-unit", '~> 3.2'
+  s.add_development_dependency 'bundler', '~> 1.16'
+  s.add_development_dependency 'checkm', '0.0.6'
+  s.add_development_dependency 'mocha', '~> 1.7'
+  s.add_development_dependency 'rake', '~> 12.0'
+  s.add_development_dependency 'rspec', '~> 3.8'
+  s.add_development_dependency 'rubocop', '~> 0.68'
+  s.add_development_dependency 'shoulda', '~> 3.6'
+  s.add_development_dependency 'simplecov', '~> 0.16'
+  s.add_development_dependency 'simplecov-console', '~> 0.4'
+  s.add_development_dependency 'webmock', '~> 3.5'
 
   s.files         = `git ls-files -z`.split("\x0")
   s.test_files    = s.files.grep(%r{^(test|spec|features)/})
   s.executables   = s.files.grep(%r{^bin/}) { |f| File.basename(f) }
-  s.require_paths = ["lib"]
+  s.require_paths = ['lib']
 end

data/spec/.rubocop.yml

@@ -0,0 +1,16 @@
+inherit_from: ../.rubocop.yml
+
+Security/Open:
+  Enabled: false
+
+Style/ClassAndModuleChildren:
+  Enabled: false
+
+Metrics/BlockLength:
+  Enabled: false
+
+Metrics/ModuleLength:
+  Enabled: false
+
+Metrics/MethodLength:
+  Enabled: false

data/spec/spec_helper.rb

@@ -0,0 +1,36 @@
+# ------------------------------------------------------------
+# SimpleCov setup
+
+if ENV['COVERAGE']
+  require 'simplecov'
+  require 'simplecov-console'
+
+  SimpleCov.command_name 'spec:lib'
+
+  SimpleCov.minimum_coverage 100
+  SimpleCov.start do
+    add_filter '/spec/'
+    SimpleCov.formatter = SimpleCov::Formatter::MultiFormatter[
+      SimpleCov::Formatter::HTMLFormatter,
+      SimpleCov::Formatter::Console,
+    ]
+  end
+end
+
+# ------------------------------------------------------------
+# Rspec configuration
+
+require 'webmock/rspec'
+
+RSpec.configure do |config|
+  config.raise_errors_for_deprecations!
+  config.mock_with :rspec
+  hostname_lower = Socket.gethostname.downcase # https://github.com/bblimke/webmock/issues/819
+  config.before(:each) { WebMock.disable_net_connect!(allow: hostname_lower) }
+  config.after(:each) { WebMock.allow_net_connect! }
+end
+
+# ------------------------------------------------------------
+# Code under test
+
+require 'mrt/ingest'

data/spec/unit/data/file.txt

@@ -0,0 +1 @@
+I am a file.

data/spec/unit/mrt/ingest/client_spec.rb

@@ -0,0 +1,31 @@
+require 'spec_helper'
+
+module Mrt::Ingest
+  describe Client do
+    describe :new do
+      it 'creates a client' do
+        client = Client.new('http://example.org/ingest')
+        expect(client).to be_a(Client)
+      end
+
+      it 'accepts credentials' do
+        client = Client.new('http://example.org/ingest', 'me', 'secret')
+        expect(client).to be_a(Client)
+      end
+    end
+
+    describe :mk_request do
+      before(:each) do
+        @client = Client.new('http://example.org/ingest', 'me', 'secret')
+        @iobject = IObject.new
+        @ingest_req = @iobject.mk_request('profile', 'submitter')
+      end
+
+      it 'creates a valid request' do
+        rest_req = @client.mk_rest_request(@ingest_req)
+        expect(rest_req.user).to eq('me')
+        expect(rest_req.password).to eq('secret')
+      end
+    end
+  end
+end

data/spec/unit/mrt/ingest/component_spec.rb

@@ -0,0 +1,12 @@
+require 'spec_helper'
+
+module Mrt::Ingest
+  describe Component do
+    describe :from_erc do
+      it 'rejects string ERCs' do
+        server = instance_double(OneTimeServer)
+        expect { Component.from_erc(server, 'I am not an ERC') }.to raise_error(ArgumentError)
+      end
+    end
+  end
+end

data/spec/unit/mrt/ingest/iobject_spec.rb

@@ -0,0 +1,248 @@
+require 'spec_helper'
+require 'checkm'
+require 'English'
+
+module Mrt::Ingest
+  describe IObject do
+
+    RESPONSE_JSON = <<~JSON.freeze
+      {
+          "bat:batchState": {
+              "bat:batchID":"bid-8c0fa0c2-f3d7-4deb-bd49-f953f6752b59",
+              "bat:updateFlag":false,
+              "bat:targetQueue":"example.org:2181",
+              "bat:batchStatus":"QUEUED",
+              "bat:userAgent":"egh/Erik Hetzner",
+              "bat:submissionDate":"2011-08-31T15:40:26-07:00",
+              "bat:targetQueueNode":"/ingest.example.1",
+              "bat:batchProfile": {
+                  "bat:owner":"ark:/99999/fk4tt4wsh",
+                  "bat:creationDate":"2010-01-19T13:28:14-08:00",
+                  "bat:targetStorage": {
+                      "bat:storageLink":"http://example.org:35121",
+                      "bat:nodeID":10
+                  },
+                  "bat:objectType":"MRT-curatorial",
+                  "bat:modificationDate":"2010-01-26T23:28:14-08:00",
+                  "bat:aggregateType":"",
+                  "bat:objectMinterURL":"https://example.org/ezid/shoulder/ark:/99999/fk4",
+                  "bat:collection": {
+                  },
+                  "bat:profileID":"merritt_content",
+                  "bat:profileDescription":"Merritt demo content",
+                  "bat:fixityURL":"http://example.org:33143",
+                  "bat:contactsEmail": {
+                      "bat:notification": {
+                          "bat:contactEmail":"erik.hetzner@example.org"
+                      }
+                  },
+                  "bat:identifierScheme":"ARK",
+                  "bat:identifierNamespace":"99999",
+                  "bat:objectRole":"MRT-content"
+              }
+          }
+      }
+    JSON
+
+    def parse_object_manifest(iobject)
+      req = iobject.mk_request('profile', 'submitter')
+      args = req.mk_args
+      Checkm::Manifest.new(args['file'].read)
+    end
+
+    def write_to_tempfile(content)
+      tempfile = Tempfile.new('test_iobject')
+      tempfile << content
+      tempfile.open
+      tempfile
+    end
+
+    def get_uri_for_name(iobject, name)
+      manifest = parse_object_manifest(iobject)
+      manifest.entries.find do |entry|
+        entry.values[-2] == name
+      end
+    end
+
+    def parse_erc(erc)
+      arr = erc.lines.map do |line|
+        md = line.chomp.match(/^([^:]+):\s*(.*)$/)
+        [md[1], md[2]]
+      end.flatten
+      h = Hash[*arr]
+      h.delete('erc')
+      h
+    end
+
+    def parse_erc_entry(erc_entry)
+      parse_erc(open(erc_entry.values[0]).read)
+    end
+
+    def check_erc_content(iobject, asserted_erc)
+      erc_entry = get_uri_for_name(iobject, 'mrt-erc.txt')
+      expect(erc_entry).not_to be_nil
+      iobject.start_server
+      begin
+        expect(parse_erc_entry(erc_entry)).to eq(asserted_erc)
+      ensure
+        iobject.stop_server
+      end
+    end
+
+    describe 'with local ID' do
+      before(:each) do
+        @local_id = '10.1098/rstl.1665.0007'
+        @iobject = IObject.new(local_identifier: @local_id)
+      end
+
+      it 'includes the local ID in the request' do
+        request = @iobject.mk_request('profile', 'submitter')
+        expect(request.local_identifier).to eq(@local_id)
+      end
+    end
+
+    describe 'without local ID' do
+      before(:each) do
+        @iobject = IObject.new
+      end
+
+      it 'accepts a URI component' do
+        uri = URI.parse('http://example.org/file')
+        components = @iobject.add_component(uri)
+        expect(components.size).to eq(1)
+        expect(components[0].uri).to eq(uri)
+      end
+
+      it 'does not accept a string URI' do
+        expect { @iobject.add_component('http://example.org/file') }.to raise_error(ArgumentError)
+      end
+
+      it 'makes a request' do
+        req = @iobject.mk_request('profile', 'submitter')
+        expect(req.profile).to eq('profile')
+        expect(req.submitter).to eq('submitter')
+      end
+    end
+
+    describe 'the created request' do
+      before(:each) do
+        @iobject = IObject.new
+        @manifest = parse_object_manifest(@iobject)
+        @erc_entry = get_uri_for_name(@iobject, 'mrt-erc.txt')
+      end
+
+      it 'should generate a valid manifest file' do
+        expect(@manifest.entries).not_to be_empty
+      end
+
+      it 'should serve a valid mrt-erc.txt entry' do
+        expect(@erc_entry).not_to be_nil
+        @iobject.start_server
+        begin
+          open(@erc_entry.values[0]).read.lines.to_a
+        ensure
+          @iobject.stop_server
+        end
+      end
+
+      describe :erc do
+        ERC_CONTENT = <<~ERC.freeze
+          who: John Doe
+          what: Something
+          when: now
+        ERC
+
+        it 'should accept a file' do
+          erc_tempfile = write_to_tempfile(ERC_CONTENT)
+          iobject = IObject.new(erc: File.new(erc_tempfile.path))
+          check_erc_content(iobject, parse_erc(ERC_CONTENT))
+        end
+
+        it 'should accept a hash' do
+          erc = {
+            'who' => 'John Doe',
+            'what' => 'Something',
+            'when' => 'now'
+          }
+          iobject = Mrt::Ingest::IObject.new(erc: erc)
+          check_erc_content(iobject, erc)
+        end
+      end
+
+      describe 'local files' do
+        FILE_CONTENT = <<~TXT.freeze
+          Hello, world!
+        TXT
+
+        FILE_CONTENT_MD5 = '746308829575e17c3331bbcb00c0898b'.freeze
+
+        it 'should support local files' do
+          iobject = IObject.new
+          tempfile = write_to_tempfile(FILE_CONTENT)
+          iobject.add_component(tempfile, { name: 'helloworld' })
+          uri_entry = get_uri_for_name(iobject, 'helloworld')
+          erc_entry = get_uri_for_name(iobject, 'mrt-erc.txt')
+          expect(erc_entry).not_to(be_nil)
+          manifest = parse_object_manifest(iobject)
+          expect(manifest).not_to(be_nil)
+          expect(uri_entry).not_to be_nil
+          iobject.start_server
+          begin
+            expect(open(uri_entry.values[0]).read).to eq(FILE_CONTENT)
+          ensure
+            iobject.stop_server
+          end
+        end
+      end
+    end
+
+    describe :start_ingest do
+      it 'submits the object' do
+        @ingest_url = 'http://merritt.example.edu:33121/poster/submit/'
+        @client = Client.new(@ingest_url)
+
+        @iobject = IObject.new
+        @iobject.add_component(URI.parse('http://example.org'), name: 'index.html')
+
+        stub_request(:post, 'http://merritt.example.edu:33121/poster/submit/').to_return(status: 200, body: RESPONSE_JSON, headers: {})
+        @iobject.start_ingest(@client, 'example_profile', 'Atom processor/Example collection')
+      end
+    end
+
+    describe :finish_ingest do
+      it 'joins the server' do
+        @ingest_url = 'http://merritt.example.edu:33121/poster/submit/'
+        @client = Client.new(@ingest_url)
+
+        @iobject = IObject.new
+        @iobject.add_component(URI.parse('http://example.org'), name: 'index.html')
+
+        stub_request(:post, 'http://merritt.example.edu:33121/poster/submit/').to_return(status: 200, body: RESPONSE_JSON, headers: {})
+        @iobject.start_ingest(@client, 'example_profile', 'Atom processor/Example collection')
+
+        # TODO: just mock the server
+        server = @iobject.server
+        files = Dir.entries(server.dir).reject { |e| %w[. ..].include?(e) }
+        urls = files.map { |f| "http://#{Socket.gethostname}:#{server.port}/#{f}" }
+
+        client_process_id = fork do
+          begin
+            urls.each do |url|
+              resp = Net::HTTP.get_response(URI.parse(url))
+              status = resp.code.to_i
+              exit(status) if status != 200
+            end
+          rescue StandardError => e
+            warn(e)
+            exit(1)
+          end
+        end
+        Process.wait(client_process_id)
+        expect($CHILD_STATUS.exitstatus).to eq(0) # just to be sure
+
+        @iobject.finish_ingest
+      end
+    end
+
+  end
+end