mrt-ingest 0.0.6 → 0.0.10

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: c46977a557350af94d24aea5e4d53ed2252aa13bdbb059289ac48b6c36c9a05c
-  data.tar.gz: bb12de3c86fe0a45f9b7f4fbea5f0a9478d13ed5e5dd1f6dea09da655cd08306
+  metadata.gz: 45bcaa8222c9328220d0a78f058217d7d75f74bbfa3665d2b4575f6331871795
+  data.tar.gz: e14e16b3101143fff600664d7178069b48f1a0a2c77582eb11942c0124486630
 SHA512:
-  metadata.gz: 5391b059eb9bcfca59bc4af506b4c8cddf5a923e53b23e35f12f255893c5e88e178e2fdff150555a37477a0c4e64b45f9d3cf71a61855a03b7730a1a0084cac5
-  data.tar.gz: 28cef0d3f22950f7127e766a2e1de0ebd78bffd326ee3a85e0a66beeb790b7417860155b61221dce09e932352c3411368294a467026f0e60e425ef0351588147
+  metadata.gz: cf9e18c9d0d6cc47f7d238d5726714991f3ff2a8201ca9026516843f43d964fa8f9bb4099a668a21b20da0d4661ed5a0a893707648ce0b67650f4288d3a044d3
+  data.tar.gz: ee160153eb825804c5a53080fc4b09562bc6dcd0bdc4527705efaf1706c3cd7ced461e0edb8f22c8a7ea47c47e3e271d83685430c2c93272610dcb6e7e859ecd

data/lib/mrt/ingest/component.rb

@@ -6,10 +6,9 @@ module Mrt
     # #File.
     class Component # :nodoc:
 
-      attr_reader :server, :uri
+      attr_reader :uri
 
-      def initialize(server, location, options)
-        @server = server
+      def initialize(location, options)
         @name = options[:name]
         @digest = options[:digest]
         @mime_type = options[:mime_type]
@@ -18,25 +17,6 @@ module Mrt
         init_uri(location)
       end
 
-      class << self
-        def from_erc(server, erc)
-          return Component.new(server, erc, name: 'mrt-erc.txt') if erc.is_a?(URI) || erc.is_a?(File)
-          return from_hash(server, erc) if erc.is_a?(Hash)
-
-          raise ArgumentError, 'Bad ERC supplied: must be a URI, File, or Hash'
-        end
-
-        def from_hash(server, erc_h)
-          uri_str, path = server.add_file do |f|
-            f.write("erc:\n")
-            erc_h.each_pair { |k, v| f.write("#{k}: #{v}\n") }
-          end
-
-          digest = Mrt::Ingest::MessageDigest::MD5.from_file(File.new(path))
-          Component.new(server, URI.parse(uri_str), name: 'mrt-erc.txt', digest: digest)
-        end
-      end
-
       def to_manifest_entry
         "#{@uri} | #{digest_type} | #{digest_value} | #{@size} | | #{@name} | #{@mime_type}\n"
       end
@@ -64,7 +44,7 @@ module Mrt
 
       def init_from_file(file)
         @name = File.basename(file.path) if @name.nil?
-        @uri = server.add_file(file)[0]
+        # @uri = server.add_file(file)[0]
         @digest = Mrt::Ingest::MessageDigest::MD5.from_file(file) if @digest.nil?
         @size = File.size(file.path) if @size.nil?
       end

data/lib/mrt/ingest/iobject.rb

@@ -12,18 +12,16 @@ module Mrt
     # An object prepared for ingest into Merritt.
     class IObject
 
-      attr_accessor :primary_identifier, :local_identifier, :erc
-      attr_reader :server
+      attr_accessor :primary_identifier, :local_identifier, :erc, :what, :who, :when
 
       # Options can have the keys :primary_identifier,
-      # :local_identifier, :server, or :erc. :erc can be a #File, #Uri
-      # or a #Hash of metadata. :server is a #OneTimeServer.
+      # :local_identifier, or :erc. :erc can be a #File, #Uri
+      # or a #Hash of metadata. 
       def initialize(options = {})
         @primary_identifier = options[:primary_identifier]
         @local_identifier = options[:local_identifier]
         @erc = options[:erc] || {}
         @components = []
-        @server = options[:server] || Mrt::Ingest::OneTimeServer.new
       end
 
       # Add a component to the object. where can be either a #URI or a
@@ -32,33 +30,20 @@ module Mrt
       # subclass of Mrt::Ingest::MessageDigest::Base. If where is a
       # #File, it will be hosted on an embedded web server.
       def add_component(where, options = {})
-        @components.push(Component.new(@server, where, options))
+        @components.push(Component.new(where, options))
       end
 
       # Make a Mrt::Ingest::Request object for this mrt-object
       def mk_request(profile, user_agent)
         manifest_file = Tempfile.new('mrt-ingest')
-        erc_component = Component.from_erc(@server, @erc)
-        mk_manifest(manifest_file, erc_component)
+        mk_manifest(manifest_file)
         # reset to beginning
         manifest_file.open
         new_request(manifest_file, profile, user_agent)
       end
 
-      def start_server # :nodoc:
-        @server.start_server
-      end
-
-      def join_server # :nodoc:
-        @server.join_server
-      end
-
-      def stop_server # :nodoc:
-        @server.stop_server
-      end
-
       # rubocop:disable Metrics/LineLength
-      def mk_manifest(manifest, erc_component) # :nodoc:
+      def mk_manifest(manifest) # :nodoc:
         manifest.write("#%checkm_0.7\n")
         manifest.write("#%profile http://uc3.cdlib.org/registry/ingest/manifest/mrt-ingest-manifest\n")
         manifest.write("#%prefix | mrt: | http://uc3.cdlib.org/ontology/mom#\n")
@@ -67,7 +52,6 @@ module Mrt
         @components.each do |c|
           manifest.write(c.to_manifest_entry)
         end
-        manifest.write(erc_component.to_manifest_entry)
         manifest.write("#%EOF\n")
       end
       # rubocop:enable Metrics/LineLength
@@ -76,7 +60,6 @@ module Mrt
       # submitter.
       def start_ingest(client, profile, submitter)
         request = mk_request(profile, submitter)
-        start_server
         @response = client.ingest(request)
       end
 
@@ -84,7 +67,6 @@ module Mrt
       def finish_ingest
         # XXX Right now we only join the hosting server; in the future
         # we will check the status via the ingest server.
-        join_server
       end
 
       private
@@ -96,6 +78,9 @@ module Mrt
           type: 'object-manifest',
           submitter: user_agent,
           profile: profile,
+          title: @erc['what'],
+          creator: @erc['who'],
+          date: @erc['when'],
           local_identifier: @local_identifier,
           primary_identifier: @primary_identifier
         )

data/mrt-ingest.gemspec

@@ -3,7 +3,7 @@ $LOAD_PATH.push File.expand_path('lib', __dir__)
 Gem::Specification.new do |s|
   s.required_ruby_version = '>= 2.4.0'
   s.name        = 'mrt-ingest'
-  s.version     = '0.0.6'
+  s.version     = '0.0.10'
   s.platform    = Gem::Platform::RUBY
   s.authors     = ['Mark Reyes', 'David Moles']
   s.email       = ['mark.reyes@ucop.edu', 'david.moles@ucop.edu']
@@ -12,10 +12,11 @@ Gem::Specification.new do |s|
   s.description = 'A client for the Merritt ingest system. More details available from https://github.com/CDLUC3/mrt-doc/wiki'
   s.license     = 'BSD-3-Clause'
 
-  s.add_dependency 'json', '~> 2.0'
-  s.add_dependency 'rest-client', '~> 2.0'
+  # s.add_dependency 'json', '~> 2.1'
+  s.add_dependency 'rest-client', '~> 2.1'
 
-  s.add_development_dependency 'bundler', '>= 2.2.10'
+
+  # s.add_development_dependency 'bundler', '>= 2.2.10'
   s.add_development_dependency 'checkm', '0.0.6'
   s.add_development_dependency 'mocha', '~> 1.7'
   s.add_development_dependency 'rake', '~> 12.0'

data/spec/unit/mrt/ingest/component_spec.rb

@@ -4,8 +4,6 @@ module Mrt::Ingest
   describe Component do
     describe :from_erc do
       it 'rejects string ERCs' do
-        server = instance_double(OneTimeServer)
-        expect { Component.from_erc(server, 'I am not an ERC') }.to raise_error(ArgumentError)
       end
     end
   end

data/spec/unit/mrt/ingest/iobject_spec.rb

@@ -81,11 +81,9 @@ module Mrt::Ingest
     def check_erc_content(iobject, asserted_erc)
       erc_entry = get_uri_for_name(iobject, 'mrt-erc.txt')
       expect(erc_entry).not_to be_nil
-      iobject.start_server
       begin
         expect(parse_erc_entry(erc_entry)).to eq(asserted_erc)
       ensure
-        iobject.stop_server
       end
     end
 
@@ -137,11 +135,9 @@ module Mrt::Ingest
 
       it 'should serve a valid mrt-erc.txt entry' do
         expect(@erc_entry).not_to be_nil
-        @iobject.start_server
         begin
           open(@erc_entry.values[0]).read.lines.to_a
         ensure
-          @iobject.stop_server
         end
       end
 
@@ -186,11 +182,9 @@ module Mrt::Ingest
           manifest = parse_object_manifest(iobject)
           expect(manifest).not_to(be_nil)
           expect(uri_entry).not_to be_nil
-          iobject.start_server
           begin
             expect(open(uri_entry.values[0]).read).to eq(FILE_CONTENT)
           ensure
-            iobject.stop_server
           end
         end
       end
@@ -221,9 +215,8 @@ module Mrt::Ingest
         @iobject.start_ingest(@client, 'example_profile', 'Atom processor/Example collection')
 
         # TODO: just mock the server
-        server = @iobject.server
-        files = Dir.entries(server.dir).reject { |e| %w[. ..].include?(e) }
-        urls = files.map { |f| "http://#{Socket.gethostname}:#{server.port}/#{f}" }
+        # files = Dir.entries(server.dir).reject { |e| %w[. ..].include?(e) }
+        # urls = files.map { |f| "http://#{Socket.gethostname}:#{server.port}/#{f}" }
 
         client_process_id = fork do
           begin

metadata

@@ -1,58 +1,30 @@
 --- !ruby/object:Gem::Specification
 name: mrt-ingest
 version: !ruby/object:Gem::Version
-  version: 0.0.6
+  version: 0.0.10
 platform: ruby
 authors:
 - Mark Reyes
 - David Moles
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-06-08 00:00:00.000000000 Z
+date: 2022-07-12 00:00:00.000000000 Z
 dependencies:
-- !ruby/object:Gem::Dependency
-  name: json
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '2.0'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '2.0'
 - !ruby/object:Gem::Dependency
   name: rest-client
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.0'
+        version: '2.1'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.0'
-- !ruby/object:Gem::Dependency
-  name: bundler
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: 2.2.10
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: 2.2.10
+        version: '2.1'
 - !ruby/object:Gem::Dependency
   name: checkm
   requirement: !ruby/object:Gem::Requirement
@@ -209,7 +181,6 @@ files:
 - lib/mrt/ingest/ingest_exception.rb
 - lib/mrt/ingest/iobject.rb
 - lib/mrt/ingest/message_digest.rb
-- lib/mrt/ingest/one_time_server.rb
 - lib/mrt/ingest/request.rb
 - lib/mrt/ingest/response.rb
 - mrt-ingest-ruby.iml
@@ -221,14 +192,13 @@ files:
 - spec/unit/mrt/ingest/component_spec.rb
 - spec/unit/mrt/ingest/iobject_spec.rb
 - spec/unit/mrt/ingest/message_digest_spec.rb
-- spec/unit/mrt/ingest/one_time_server_spec.rb
 - spec/unit/mrt/ingest/request_spec.rb
 - spec/unit/mrt/ingest/response_spec.rb
 homepage: https://github.com/CDLUC3/mrt-ingest-ruby
 licenses:
 - BSD-3-Clause
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -244,7 +214,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubygems_version: 3.0.3.1
-signing_key: 
+signing_key:
 specification_version: 4
 summary: A client for Merritt ingest.
 test_files:
@@ -255,6 +225,5 @@ test_files:
 - spec/unit/mrt/ingest/component_spec.rb
 - spec/unit/mrt/ingest/iobject_spec.rb
 - spec/unit/mrt/ingest/message_digest_spec.rb
-- spec/unit/mrt/ingest/one_time_server_spec.rb
 - spec/unit/mrt/ingest/request_spec.rb
 - spec/unit/mrt/ingest/response_spec.rb

data/lib/mrt/ingest/one_time_server.rb

@@ -1,111 +0,0 @@
-# Author::    Erik Hetzner  (mailto:erik.hetzner@ucop.edu)
-# Copyright:: Copyright (c) 2011, Regents of the University of California
-
-require 'webrick'
-
-# An HTTP server that will serve each file ONCE before shutting down.
-module Mrt
-  module Ingest
-    class OneTimeServer
-
-      attr_reader :dir, :port
-
-      # Find an open port, starting with start and adding one until we get
-      # an open port
-      def get_open_port(start = 8081)
-        try_port = start
-        loop do
-          begin
-            s = TCPServer.open(try_port)
-            s.close
-            return try_port
-          rescue Errno::EADDRINUSE
-            try_port += 1
-          end
-        end
-      end
-
-      def initialize
-        @dir = Dir.mktmpdir
-        @mutex = Mutex.new
-        @known_paths = {}
-        @requested = {}
-        @port = get_open_port
-        @file_callback = ->(req, _res) { @requested[req.path] ||= true }
-        @server = WEBrick::HTTPServer.new(Port: @port)
-        @server.mount('/', WEBrick::HTTPServlet::FileHandler, @dir, FileCallback: @file_callback)
-      end
-
-      # Return true if each file has been served.
-      def finished?
-        Dir.entries(@dir).each do |entry|
-          next if %w[. ..].include?(entry)
-          return false if @requested["/#{entry}"].nil?
-        end
-        true
-      end
-
-      def temppath
-        tmpfile = Tempfile.new('tmp', @dir)
-        tmppath = tmpfile.path
-        tmpfile.close!
-        @mutex.synchronize do
-          unless @known_paths.key?(tmppath)
-            # no collision
-            @known_paths[tmppath] = true
-            return tmppath
-          end
-        end
-        # need to retry, there was a collision
-        temppath
-      end
-
-      # Add a file to this server. Returns the URL to use
-      # to fetch the file & the file path
-      def add_file(sourcefile = nil)
-        fullpath = temppath
-        path = File.basename(fullpath)
-
-        if sourcefile
-          @server.mount("/#{path}", WEBrick::HTTPServlet::FileHandler, sourcefile.path, FileCallback: @file_callback)
-        else
-          File.open(fullpath, 'w+') { |f| yield f }
-        end
-        ["http://#{Socket.gethostname}:#{@port}/#{path}", fullpath]
-      end
-
-      def start_server
-        if @thread.nil?
-          @thread = Thread.new do
-            @server.start
-          end
-        end
-        sleep(0.1) while @server.status != :Running
-        @thread
-      end
-
-      # Stop server unconditionally.
-      def stop_server
-        @server.shutdown
-        @thread.join
-      end
-
-      # Wait for server to finish serving all files.
-      def join_server
-        # ensure that each file is requested once before shutting down
-        sleep(1) until finished?
-        @server.shutdown
-        @thread.join
-      end
-
-      # Run the server and wait until each file has been served once.
-      # Cleans up files before it returns.
-      def run
-        start_server
-        join_server
-        #    FileUtils.rm_rf(@dir)
-        nil
-      end
-    end
-  end
-end

data/spec/unit/mrt/ingest/one_time_server_spec.rb

@@ -1,113 +0,0 @@
-require 'spec_helper'
-require 'English'
-
-module Mrt::Ingest
-  describe OneTimeServer do
-    attr_reader :server
-
-    before(:each) do
-      @server = OneTimeServer.new
-      server.start_server
-    end
-
-    after(:each) do
-      server.stop_server
-    end
-
-    describe :finished? do
-      it 'returns true when all files have been served, false otherwise' do
-        urls = (0..3).map do |i|
-          url_str, = server.add_file { |f| f.puts("I am file #{i}") }
-          url_str
-        end
-
-        urls.each do |url|
-          expect(server.finished?).to be_falsey
-          Net::HTTP.get(URI.parse(url))
-        end
-
-        expect(server.finished?).to be_truthy
-      end
-    end
-
-    describe :temppath do
-      it 'avoids collisions' do
-        tmpfiles = []
-        allow(Tempfile).to receive(:new).and_wrap_original do |m, *args|
-          tmpfile = m.call(*args)
-          if tmpfiles.empty?
-            known_paths = server.instance_variable_get(:@known_paths)
-            known_paths[tmpfile.path] = true
-          end
-          tmpfiles << tmpfile.path
-          tmpfile
-        end
-
-        temppath = server.temppath
-        expect(tmpfiles.size).to eq(2)
-        expect(temppath).to eq(tmpfiles[1])
-      end
-    end
-
-    describe :join_server do
-      it 'blocks till all files have been served' do
-        urls = (0..3).map do |i|
-          url_str, = server.add_file { |f| f.puts("I am file #{i}") }
-          url_str
-        end
-
-        joining_thread = Thread.new { server.join_server }
-        expect(joining_thread.status).not_to be_falsey
-
-        client_process_id = fork do
-          begin
-            urls.each do |url|
-              resp = Net::HTTP.get_response(URI.parse(url))
-              status = resp.code.to_i
-              exit(status) if status != 200
-            end
-          rescue StandardError => e
-            warn(e)
-            exit(1)
-          end
-        end
-        Process.wait(client_process_id)
-        expect($CHILD_STATUS.exitstatus).to eq(0) # just to be sure
-
-        Timeout.timeout(5) { joining_thread.join }
-        expect(joining_thread.status).to eq(false)
-      end
-    end
-
-    describe :run do
-      it 'starts, serves, and stops' do
-        server2 = OneTimeServer.new
-        urls = (0..3).map do |i|
-          url_str, = server2.add_file { |f| f.puts("I am file #{i}") }
-          url_str
-        end
-
-        running_thread = Thread.new { server2.run }
-        expect(running_thread.status).not_to be_falsey
-
-        client_process_id = fork do
-          begin
-            urls.each do |url|
-              resp = Net::HTTP.get_response(URI.parse(url))
-              status = resp.code.to_i
-              exit(status) if status != 200
-            end
-          rescue StandardError => e
-            warn(e)
-            exit(1)
-          end
-        end
-        Process.wait(client_process_id)
-        expect($CHILD_STATUS.exitstatus).to eq(0) # just to be sure
-
-        Timeout.timeout(5) { running_thread.join }
-        expect(running_thread.status).to eq(false)
-      end
-    end
-  end
-end