mrt-ingest 0.0.4 → 0.0.5

data/lib/mrt/ingest/ingest_exception.rb

@@ -0,0 +1,6 @@
+module Mrt
+  module Ingest
+    class IngestException < RuntimeError
+    end
+  end
+end

data/lib/mrt/ingest/iobject.rb

@@ -1,7 +1,6 @@
 # Author::    Erik Hetzner  (mailto:erik.hetzner@ucop.edu)
 # Copyright:: Copyright (c) 2011, Regents of the University of California
 
-require 'mrt/ingest'
 require 'tempfile'
 require 'uri'
 require 'open-uri'
@@ -9,147 +8,76 @@ require 'digest/md5'
 
 module Mrt
   module Ingest
-    # Represents a component of an object to ingest. Either a #URI or a
-    # #File.
-    class Component # :nodoc:
-      def initialize(server, where, options)
-        @name = options[:name]
-        @digest = options[:digest]
-        @mime_type = options[:mime_type]
-        @size = options[:size]
-        # @prefetch = options[:prefetch] || false
-        @prefetch = false # TODO: remove prefetch code
-
-        case where
-        when File, Tempfile
-          @name = File.basename(where.path) if @name.nil?
-          @uri = server.add_file(where)[0]
-          if @digest.nil? then
-            @digest = Mrt::Ingest::MessageDigest::MD5.from_file(where)
-          end
-          @size = File.size(where.path) if @size.nil?
-        when URI
-          @name = File.basename(where.to_s) if @name.nil?
-          if @prefetch then
-            digest = Digest::MD5.new()
-            @uri, ignore = server.add_file do |f|
-              open(where, (options[:prefetch_options] || {})) do |u|
-                while (buff = u.read(1024)) do
-                  f << buff
-                  digest << buff
-                end
-              end
-            end
-            @digest = Mrt::Ingest::MessageDigest::MD5.new(digest.hexdigest)
-          else
-            @uri = where
-          end
-        else
-          raise IngestException.new("Trying to add a component that is not a File or URI")
-        end
-        
-      end
-      
-      def to_manifest_entry
-        (digest_alg, digest_value) = if @digest.nil? then
-                                       ['', '']
-                                     else
-                                       [@digest.type, @digest.value]
-                                     end
-        return "#{@uri} | #{digest_alg} | #{digest_value} | #{@size || ''} | | #{@name} | #{@mime_type || '' }\n"
-      end
-    end
-    
+
     # An object prepared for ingest into Merritt.
     class IObject
-      
+
       attr_accessor :primary_identifier, :local_identifier, :erc
+      attr_reader :server
 
       # Options can have the keys :primary_identifier,
       # :local_identifier, :server, or :erc. :erc can be a #File, #Uri
       # or a #Hash of metadata. :server is a #OneTimeServer.
-      def initialize(options={})
+      def initialize(options = {})
         @primary_identifier = options[:primary_identifier]
         @local_identifier = options[:local_identifier]
-        @erc = options[:erc] || Hash.new
+        @erc = options[:erc] || {}
         @components = []
         @server = options[:server] || Mrt::Ingest::OneTimeServer.new
       end
-      
+
       # Add a component to the object. where can be either a #URI or a
       # #File. Options is a hash whose keys may be :name, :digest,
       # :mime_type, or :size. If :digest is supplied, it must be a
       # subclass of Mrt::Ingest::MessageDigest::Base. If where is a
       # #File, it will be hosted on an embedded web server.
-      def add_component(where, options={})
+      def add_component(where, options = {})
         @components.push(Component.new(@server, where, options))
       end
-      
+
       # Make a Mrt::Ingest::Request object for this mrt-object
-      def mk_request(profile, submitter)
-        erc_component = case @erc
-                        when URI, File, Tempfile
-                          Component.new(@server, @erc, :name => 'mrt-erc.txt')
-                        when Hash
-                          uri_str, path = @server.add_file do |f|
-                            f.write("erc:\n")
-                            @erc.each_pair do |k, v|
-                              f.write("#{k}: #{v}\n")
-                            end
-                          end
-                          Component.new(@server, 
-                                        URI.parse(uri_str), 
-                                        :name => 'mrt-erc.txt',
-                                        :digest => Mrt::Ingest::MessageDigest::MD5.from_file(File.new(path)))
-                        else
-                          raise IngestException.new("Bad ERC supplied: must be a URI, File, or Hash")
-                        end
-        manifest_file = Tempfile.new("mrt-ingest")
+      def mk_request(profile, user_agent)
+        manifest_file = Tempfile.new('mrt-ingest')
+        erc_component = Component.from_erc(@server, @erc)
         mk_manifest(manifest_file, erc_component)
         # reset to beginning
         manifest_file.open
-        return Mrt::Ingest::Request.
-          new(:file               => manifest_file,
-              :filename           => manifest_file.path.split(/\//).last,
-              :type               => "object-manifest",
-              :submitter          => submitter,
-              :profile            => profile,
-              :local_identifier => @local_identifier,
-              :primary_identifier => @primary_identifier)
+        new_request(manifest_file, profile, user_agent)
       end
 
       def start_server # :nodoc:
-        return @server.start_server()
+        @server.start_server
       end
 
       def join_server # :nodoc:
-        return @server.join_server()
+        @server.join_server
       end
 
       def stop_server # :nodoc:
-        return @server.stop_server()
+        @server.stop_server
       end
-        
+
+      # rubocop:disable Metrics/LineLength
       def mk_manifest(manifest, erc_component) # :nodoc:
         manifest.write("#%checkm_0.7\n")
         manifest.write("#%profile http://uc3.cdlib.org/registry/ingest/manifest/mrt-ingest-manifest\n")
         manifest.write("#%prefix | mrt: | http://uc3.cdlib.org/ontology/mom#\n")
         manifest.write("#%prefix | nfo: | http://www.semanticdesktop.org/ontologies/2007/03/22/nfo#\n")
         manifest.write("#%fields | nfo:fileUrl | nfo:hashAlgorithm | nfo:hashValue | nfo:fileSize | nfo:fileLastModified | nfo:fileName | mrt:mimeType\n")
-        @components.each { |c|
+        @components.each do |c|
           manifest.write(c.to_manifest_entry)
-        }
+        end
         manifest.write(erc_component.to_manifest_entry)
         manifest.write("#%EOF\n")
       end
-      
+      # rubocop:enable Metrics/LineLength
+
       # Begin an ingest on the given client, with a profile and
       # submitter.
       def start_ingest(client, profile, submitter)
         request = mk_request(profile, submitter)
         start_server
         @response = client.ingest(request)
-        return @response
       end
 
       # Wait for the ingest of this object to finish.
@@ -158,6 +86,21 @@ module Mrt
         # we will check the status via the ingest server.
         join_server
       end
+
+      private
+
+      def new_request(manifest_file, profile, user_agent)
+        Mrt::Ingest::Request.new(
+          file: manifest_file,
+          filename: manifest_file.path.split(%r{/}).last,
+          type: 'object-manifest',
+          submitter: user_agent,
+          profile: profile,
+          local_identifier: @local_identifier,
+          primary_identifier: @primary_identifier
+        )
+      end
+
     end
   end
 end

data/lib/mrt/ingest/message_digest.rb

@@ -13,37 +13,35 @@ module Mrt
           @type  = type
         end
       end
-      
+
       # Represents a SHA256 digest suitable for a Checkm manifest.
       class SHA256 < Base
         def initialize(value)
-          super(value, "sha-256")
+          super(value, 'sha-256')
         end
       end
 
       # Represents an MD5 digest suitable for a Checkm manifest.
       class MD5 < Base
         def initialize(value)
-          super(value, "md5")
+          super(value, 'md5')
         end
-        
+
         # Generate a digest from a file.
         def self.from_file(file)
           digest = Digest::MD5.new
           File.open(file.path, 'r') do |f|
-            buff = ""
-            while (f.read(1024, buff) != nil)
-              digest << buff
-            end
+            buff = ''
+            digest << buff until f.read(1024, buff).nil?
           end
-          return Mrt::Ingest::MessageDigest::MD5.new(digest.hexdigest)
+          Mrt::Ingest::MessageDigest::MD5.new(digest.hexdigest)
         end
       end
 
       # Represents a SHA1 digest suitable for a Checkm manifest.
       class SHA1 < Base
         def initialize(value)
-          super(value, "sha1")
+          super(value, 'sha1')
         end
       end
     end

data/lib/mrt/ingest/one_time_server.rb

@@ -7,17 +7,20 @@ require 'webrick'
 module Mrt
   module Ingest
     class OneTimeServer
+
+      attr_reader :dir, :port
+
       # Find an open port, starting with start and adding one until we get
       # an open port
-      def get_open_port(start=8081)
+      def get_open_port(start = 8081)
         try_port = start
-        while (true)
+        loop do
           begin
             s = TCPServer.open(try_port)
             s.close
             return try_port
           rescue Errno::EADDRINUSE
-            try_port = try_port + 1
+            try_port += 1
           end
         end
       end
@@ -27,71 +30,58 @@ module Mrt
         @mutex = Mutex.new
         @known_paths = {}
         @requested = {}
-        @port = get_open_port()
-        @file_callback = lambda do |req, res|
-          @requested[req.path] ||= true
-        end
-      
-        config = { :Port => @port }
-        @server = WEBrick::HTTPServer.new(config)
-        @server.mount("/", WEBrick::HTTPServlet::FileHandler, @dir,
-                      { :FileCallback=>@file_callback })
+        @port = get_open_port
+        @file_callback = ->(req, _res) { @requested[req.path] ||= true }
+        @server = WEBrick::HTTPServer.new(Port: @port)
+        @server.mount('/', WEBrick::HTTPServlet::FileHandler, @dir, FileCallback: @file_callback)
       end
 
       # Return true if each file has been served.
       def finished?
         Dir.entries(@dir).each do |entry|
-          next if (entry == "." || entry == "..")
-          if @requested["/#{entry}"].nil? then
-            return false
-          end
+          next if %w[. ..].include?(entry)
+          return false if @requested["/#{entry}"].nil?
         end
-        return true
+        true
       end
 
-      def get_temppath
-        tmpfile = Tempfile.new("tmp", @dir)
+      def temppath
+        tmpfile = Tempfile.new('tmp', @dir)
         tmppath = tmpfile.path
         tmpfile.close!
         @mutex.synchronize do
-          if !@known_paths.has_key?(tmppath) then
+          unless @known_paths.key?(tmppath)
             # no collision
             @known_paths[tmppath] = true
             return tmppath
           end
         end
         # need to retry, there was a collision
-        return get_temppath
+        temppath
       end
 
       # Add a file to this server. Returns the URL to use
       # to fetch the file & the file path
-      def add_file(sourcefile=nil)
-        fullpath = get_temppath()
+      def add_file(sourcefile = nil)
+        fullpath = temppath
         path = File.basename(fullpath)
-        if !sourcefile.nil? then
-          @server.mount("/#{path}",
-                        WEBrick::HTTPServlet::FileHandler,
-                        sourcefile.path,
-                        { :FileCallback=>@file_callback })
+
+        if sourcefile
+          @server.mount("/#{path}", WEBrick::HTTPServlet::FileHandler, sourcefile.path, FileCallback: @file_callback)
         else
-          File.open(fullpath, 'w+') do |f|
-            yield f
-          end
+          File.open(fullpath, 'w+') { |f| yield f }
         end
-        return "http://#{Socket.gethostname}:#{@port}/#{path}", fullpath
+        ["http://#{Socket.gethostname}:#{@port}/#{path}", fullpath]
       end
-      
+
       def start_server
-        if @thread.nil? then
+        if @thread.nil?
           @thread = Thread.new do
             @server.start
           end
         end
-        while (@server.status != :Running) do
-          sleep(0.1)
-        end
-        return @thread
+        sleep(0.1) while @server.status != :Running
+        @thread
       end
 
       # Stop server unconditionally.
@@ -103,18 +93,18 @@ module Mrt
       # Wait for server to finish serving all files.
       def join_server
         # ensure that each file is requested once before shutting down
-        while (!self.finished?) do sleep(1) end
-        @server.shutdown 
+        sleep(1) until finished?
+        @server.shutdown
         @thread.join
       end
-      
+
       # Run the server and wait until each file has been served once.
       # Cleans up files before it returns.
       def run
-        start_server()
-        join_server()
+        start_server
+        join_server
         #    FileUtils.rm_rf(@dir)
-        return
+        nil
       end
     end
   end

data/lib/mrt/ingest/request.rb

@@ -3,54 +3,80 @@
 
 module Mrt
   module Ingest
-    class RequestException < Exception
+    class RequestException < RuntimeError
     end
 
     # Represents a request to be sent to an ingest server.
     class Request
-      attr_accessor :creator, :date, :local_identifier,
-                    :primary_identifier, :profile, :note, :submitter,
-                    :title, :type
-
-      # Options is a hash; required are :profile, :submitter, :type.
-      # May also include :creator, :date, :digest, :file, :filename,
-      # :local_identifier, :primary_identifier, :note, :title.
-      def initialize(options)
-        @creator = options[:creator]
-        @date = options[:date]
-        @digest = options[:digest]
-        @file = options[:file]
-        @filename = options[:filename]
-        @local_identifier = options[:local_identifier]
-        @primary_identifier = options[:primary_identifier]
-        @profile = options[:profile]
-        @note = options[:note]
-        @submitter = options[:submitter]
-        @title = options[:title]
-        @type = options[:type]
-        [:profile, :submitter, :type].each do |arg|
-          raise RequestException.new("#{arg} is required.") if options[arg].nil?
-        end
+
+      attr_accessor :creator
+      attr_accessor :date
+      attr_accessor :digest
+      attr_accessor :file
+      attr_accessor :filename
+      attr_accessor :local_identifier
+      attr_accessor :note
+      attr_accessor :primary_identifier
+      attr_accessor :profile
+      attr_accessor :submitter
+      attr_accessor :title
+      attr_accessor :type
+
+      # rubocop:disable Metrics/MethodLength, Metrics/ParameterLists
+      def initialize(
+        profile:, submitter:, type:,
+        creator: nil, date: nil, digest: nil, file: nil, filename: nil,
+        local_identifier: nil, primary_identifier: nil, note: nil, title: nil
+      )
+        raise ArgumentError, 'profile cannot be nil' unless profile
+        raise ArgumentError, 'profile cannot be submitter' unless submitter
+        raise ArgumentError, 'profile cannot be type' unless type
+
+        @creator = creator
+        @date = date
+        @digest = digest
+        @file = file
+        @filename = filename
+        @local_identifier = local_identifier
+        @primary_identifier = primary_identifier
+        @profile = profile
+        @note = note
+        @submitter = submitter
+        @title = title
+        @type = type
       end
-      
+      # rubocop:enable Metrics/MethodLength, Metrics/ParameterLists
+
       # Returns a hash of arguments suitable for sending to a server.
+      # rubocop: disable Metrics/MethodLength, Metrics/AbcSize
       def mk_args
-        return {
-          'creator'           => @creator,
-          'date'              => @date,
-          'digestType'        => ((!@digest.nil? && @digest.type) || nil),
-          'digestValue'       => ((!@digest.nil? && @digest.value) || nil),
-          'file'              => @file,
-          'filename'          => @filename,
-          'localIdentifier'   => @local_identifier,
-          'primaryIdentifier' => @primary_identifier,
-          'profile'           => @profile,
-          'note'              => @note,
-          'responseForm'      => 'json',
-          'submitter'         => @submitter,
-          'title'             => @title,
-          'type'              => @type
-        }.reject{|k, v| v.nil? || (v == '')}
+        {
+          'creator' => creator,
+          'date' => date,
+          'digestType' => digest_type,
+          'digestValue' => digest_value,
+          'file' => file,
+          'filename' => filename,
+          'localIdentifier' => local_identifier,
+          'primaryIdentifier' => primary_identifier,
+          'profile' => profile,
+          'note' => note,
+          'responseForm' => 'json',
+          'submitter' => submitter,
+          'title' => title,
+          'type' => type
+        }.reject { |_k, v| v.nil? || (v == '') }
+      end
+      # rubocop: enable Metrics/MethodLength, Metrics/AbcSize
+
+      private
+
+      def digest_value
+        digest && digest.value
+      end
+
+      def digest_type
+        digest && digest.type
       end
     end
   end