mrt-ingest 0.0.4 → 0.0.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA256:
-  metadata.gz: b29f8fd2bc7331b638a034dc11c86726ebb3f1526a62cb2d8b71f471215110cb
-  data.tar.gz: f3797630caf8ebbda198e29590b85bdf4f01391b9d031472d5b3d950de9b836b
+SHA1:
+  metadata.gz: 564379d9be6c8e4625b31c7960d31927edf00d8c
+  data.tar.gz: 9329f69b68d2235082bc6e030d7d44529080094c
 SHA512:
-  metadata.gz: d1109ffa2df106b91e4fdef16a9b6fa2fa82305b34f56a5a189304c048490b8e56c868566f5fd2a7b64bdfdb243d65f54dd8625c059b5d1f743bd0f678af5f16
-  data.tar.gz: 9c669cbc0180ddeef7b71649cc349c804cf29dc351a140827db5a9d8a24f6a88b0889e978a7e9d48fb40522bd7c634c593e69d78d07e7692a0e2af043124525e
+  metadata.gz: '0801ccf1a414ffdbb70882036771d4e63b2138274658873e65b3df68b8b3d8e26e2ebc004d18aed3042158300ef8a0cfc21d40b10302eb4138c7f80e96fa40f6'
+  data.tar.gz: d0e324c585de1f785c8d3c54bb0217ceedb0bece034fe6694b966b72cd0cc87d57fe231c71446d9f70eef4dcffff37ac4f3e7d71393270576a858a322ebe99cb

data/.gitignore

@@ -4,6 +4,9 @@
 # Gemfile.lock should be checked in for apps, but not for libraries
 Gemfile.lock
 
+# github-markdown-preview previews
+*.md.html
+
 ###########################
 #### Generated ignores ####
 
@@ -69,7 +72,8 @@ build-iPhoneSimulator/
 .LSOverride
 
 # Icon must end with two \r
-Icon
+Icon
+
 
 # Thumbnails
 ._*

data/.idea/.rakeTasks

@@ -0,0 +1,7 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<Settings><!--This file was automatically generated by Ruby plugin.
+You are allowed to: 
+1. Remove rake task
+2. Add existing rake tasks
+To add existing rake tasks automatically delete this file and reload the project.
+--><RakeGroup description="" fullCmd="" taksId="rake"><RakeTask description="Build mrt-ingest-0.0.5.gem into the pkg directory" fullCmd="build" taksId="build" /><RakeTask description="Remove RDoc HTML files" fullCmd="clobber_rdoc" taksId="clobber_rdoc" /><RakeTask description="Build and install mrt-ingest-0.0.5.gem into system gems" fullCmd="install" taksId="install" /><RakeGroup description="" fullCmd="" taksId="install"><RakeTask description="Build and install mrt-ingest-0.0.5.gem into system gems without network access" fullCmd="install:local" taksId="local" /></RakeGroup><RakeTask description="Build RDoc HTML files" fullCmd="rdoc" taksId="rdoc" /><RakeTask description="Create tag v0.0.5 and build and push mrt-ingest-0.0.5.gem to rubygems.org" fullCmd="release[remote]" taksId="release[remote]" /><RakeTask description="Rebuild RDoc HTML files" fullCmd="rerdoc" taksId="rerdoc" /><RakeTask description="Run tests" fullCmd="test" taksId="test" /><RakeTask description="" fullCmd="clobber" taksId="clobber" /><RakeTask description="" fullCmd="default" taksId="default" /><RakeTask description="" fullCmd="html" taksId="html" /><RakeTask description="" fullCmd="html/created.rid" taksId="html/created.rid" /><RakeTask description="" fullCmd="release" taksId="release" /><RakeGroup description="" fullCmd="" taksId="release"><RakeTask description="" fullCmd="release:guard_clean" taksId="guard_clean" /><RakeTask description="" fullCmd="release:rubygem_push" taksId="rubygem_push" /><RakeTask description="" fullCmd="release:source_control_push" taksId="source_control_push" /></RakeGroup></RakeGroup></Settings>

data/.idea/encodings.xml

@@ -0,0 +1,4 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="Encoding" addBOMForNewFiles="with NO BOM" />
+</project>

data/.idea/inspectionProfiles/Project_Default.xml

@@ -0,0 +1,18 @@
+<component name="InspectionProjectProfileManager">
+  <profile version="1.0">
+    <option name="myName" value="Project Default" />
+    <inspection_tool class="ConvertOneChainedExprToSafeNavigation" enabled="true" level="INFORMATION" enabled_by_default="true" />
+    <inspection_tool class="GoNameStartsWithPackageName" enabled="false" level="WEAK WARNING" enabled_by_default="false" />
+    <inspection_tool class="GoSnakeCaseUsage" enabled="false" level="WEAK WARNING" enabled_by_default="false" />
+    <inspection_tool class="GoUnusedGlobalVariable" enabled="false" level="WEAK WARNING" enabled_by_default="false" />
+    <inspection_tool class="GoUnusedVariable" enabled="true" level="WARNING" enabled_by_default="true" />
+    <inspection_tool class="Rubocop" enabled="false" level="WARNING" enabled_by_default="false" />
+    <inspection_tool class="RubyStringKeysInHashInspection" enabled="false" level="WARNING" enabled_by_default="false" />
+    <inspection_tool class="RubyTooManyInstanceVariablesInspection" enabled="false" level="WARNING" enabled_by_default="false" />
+    <inspection_tool class="SpellCheckingInspection" enabled="false" level="TYPO" enabled_by_default="false">
+      <option name="processCode" value="true" />
+      <option name="processLiterals" value="true" />
+      <option name="processComments" value="true" />
+    </inspection_tool>
+  </profile>
+</component>

data/.idea/misc.xml

@@ -0,0 +1,32 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="GoImports">
+    <option name="optimizeImportsOnTheFly" value="false" />
+  </component>
+  <component name="NodePackageJsonFileManager">
+    <packageJsonPaths />
+  </component>
+  <component name="ProjectInspectionProfilesVisibleTreeState">
+    <entry key="Project Default">
+      <profile-state>
+        <expanded-state>
+          <State />
+          <State>
+            <id>Declaration redundancyGo</id>
+          </State>
+          <State>
+            <id>Go</id>
+          </State>
+        </expanded-state>
+        <selected-state>
+          <State>
+            <id>GoUnusedLabel</id>
+          </State>
+        </selected-state>
+      </profile-state>
+    </entry>
+  </component>
+  <component name="ProjectRootManager" version="2" languageLevel="JDK_11" default="false" project-jdk-name="RVM: ruby-2.4.4" project-jdk-type="RUBY_SDK">
+    <output url="file://$PROJECT_DIR$/out" />
+  </component>
+</project>

data/.idea/modules.xml

@@ -0,0 +1,8 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="ProjectModuleManager">
+    <modules>
+      <module fileurl="file://$PROJECT_DIR$/mrt-ingest-ruby.iml" filepath="$PROJECT_DIR$/mrt-ingest-ruby.iml" />
+    </modules>
+  </component>
+</project>

data/.idea/vcs.xml

@@ -0,0 +1,6 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="VcsDirectoryMappings">
+    <mapping directory="" vcs="Git" />
+  </component>
+</project>

data/.rubocop.yml

@@ -0,0 +1,87 @@
+AllCops:
+  # Suppress complaints about post-2.0 syntax
+  TargetRubyVersion: 2.4
+  # Exclude IDEA coverage output
+  Exclude:
+    - 'out/**/*'
+
+# Allow one line around block body (Layout/EmptyLines will still disallow two or more)
+Layout/EmptyLinesAroundBlockBody:
+  Enabled: false
+
+# Allow one line around class body (Layout/EmptyLines will still disallow two or more)
+Layout/EmptyLinesAroundClassBody:
+  Enabled: false
+
+# Allow one line around module body (Layout/EmptyLines will still disallow two or more)
+Layout/EmptyLinesAroundModuleBody:
+  Enabled: false
+
+# Make indents consistent regardless of the lengths of variables and method names and whatnot
+Layout/MultilineMethodCallIndentation:
+  EnforcedStyle: indented
+
+# Produces monsters
+Layout/MultilineOperationIndentation:
+  Enabled: false
+
+# Reasonable line-length check; it's too easy for the cure to be worse than the disease
+Metrics/LineLength:
+  Max: 150
+
+# Confusing and weird
+Naming/VariableNumber:
+  Enabled: False
+
+# We meant to do that
+Naming/MemoizedInstanceVariableName:
+  Enabled: False
+
+# It works in context, trust us
+Naming/UncommunicativeMethodParamName:
+  Enabled: False
+
+# Do what's readable in the context you're in
+Style/AccessModifierDeclarations:
+  Enabled: false
+
+# 👎 to cultural imperialism
+Style/AsciiComments:
+  Enabled: false
+
+# Sometimes we want to distinguish hash parameters from keyword arguments
+Style/BracesAroundHashParameters:
+  Enabled: False
+
+# Seriously?
+Style/CommentedKeyword:
+  Enabled: False
+
+# Disable problematic module documentation check (see https://github.com/bbatsov/rubocop/issues/947)
+Style/Documentation:
+  Enabled: false
+
+# Adding more line noise to format strings will not improve them
+Style/FormatStringToken:
+  Enabled: false
+
+# Putting '# frozen_string_literal: true' everywhere does not make the world a better place
+Style/FrozenStringLiteralComment:
+  Enabled: false
+
+# `foo.positive?` is cute, but it's not actually more readable than `foo > 0`
+Style/NumericPredicate:
+  Enabled: false
+
+# The semantics of `foo&.bar` are a lot less interchangeable with `foo && foo.bar` than RuboCop thinks
+Style/SafeNavigation:
+  Enabled: false
+
+# Requiring the lambda() method just makes wordy calls wordier
+Style/Lambda:
+  EnforcedStyle: literal
+
+# Unclear why it's a good idea to give parameters semantically meaningless names
+Style/SingleLineBlockParams:
+  Enabled: false
+

data/.ruby-version

@@ -0,0 +1,2 @@
+2.4.4
+

data/CHANGES.md

@@ -0,0 +1,25 @@
+## 0.0.5 (2019-05-17)
+
+- Invalid arguments (nil required parameters, string URLs where `URI` objects are expected, etc.) 
+  now raise `ArgumentError` instead of `IngestException`.
+- Removes prefetch option from 0.0.3, which we weren't using, and which added complexity.
+
+### Developer notes
+
+- Updates code for Ruby 2.4
+- Adds RuboCop style checks
+- Replaces Test::Unit with RSpec
+
+## 0.0.4 (2018-10-23)
+
+- Updates `json` gem from `~> 1.5` to `~> 2.0`
+- Updates `rest-client` gem from `~> 1.6` to `~> 2.0`
+
+## 0.0.3 (2018-10-23)
+
+- Adds prefetch option for remote URLs
+- Ensures one-time server is only started once
+
+## 0.0.2 (2011-12-19)
+
+- First public release

data/Gemfile

@@ -1,7 +1,4 @@
-source "http://rubygems.org"
+source 'http://rubygems.org'
 
 # Specify your gem's dependencies in mrt-ingest.gemspec
 gemspec
-
-# TODO: remove this (in favor of gemspec) once there's a 2.4-compatible release
-gem 'fakeweb', git: 'https://github.com/chrisk/fakeweb.git', ref: '2b08c1f'

data/README.md

@@ -0,0 +1,56 @@
+# mrt-ingest (ruby)
+
+[![Build Status](https://travis-ci.org/CDLUC3/mrt-ingest-ruby.svg?branch=master)](https://travis-ci.org/CDLUC3/mrt-ingest-ruby)
+
+## What?
+
+A Ruby ingest client for [Merritt](https://merritt.cdlib.org/).
+
+## Install
+
+```
+$ gem build mrt-ingest.gemspec
+$ sudo gem install mrt-ingest-0.0.1.gem
+```
+
+## Usage
+
+The code below creates a new Merritt object with content consisting of two local files
+and one remote URL. The object is submitted to Merritt as a manifest, with the manifest,
+local files, and `mrt-erc.txt` made available to Ingest by an
+[Mrt::Ingest::OneTimeServer](lib/mrt/ingest/one_time_server.rb)
+-- a simple WEBrick-based server that finds itself an open port, serves each file from
+a temporary directory, and when all files have been served, shuts down. 
+
+```ruby
+require 'mrt/ingest'
+
+client = Mrt::Ingest::Client.new(
+  "http://merritt.cdlib.org/object/ingest",
+  USERNAME,
+  PASSWORD
+)
+ingest_profile = "demo_merritt_content"
+user_agent = "me/My Name"
+
+obj = Mrt::Ingest::IObject.new(
+  erc: {
+    "who" => "Doe, John",
+    "what" => "Hello, world",
+    "when/created" => "2011"
+  }
+)
+obj.add_component(File.new("/tmp/helloworld_a"))
+obj.add_component(File.new("/tmp/helloworld_b"))
+obj.add_component(
+  URI.parse("http://example.org/xxx"),
+  name: "helloworld_c",
+  digest: Mrt::Ingest::MessageDigest::MD5.new("6f5902ac237024bdd0c176cb93063dc4")
+)
+
+obj.start_ingest(client, ingest_profile, user_agent)
+obj.finish_ingest # waits for all files to be served, then shuts down
+```
+
+For a more detailed example, see the [Merritt::Atom](https://github.com/CDLUC3/mrt-dashboard/tree/master/lib/merritt/atom)
+module of the Merritt dashboard.

data/Rakefile

@@ -1,21 +1,40 @@
-# -*- ruby -*-
-require 'rake/testtask'
-require 'rdoc/task'
-
-require 'bundler'
-include Rake::DSL
-Bundler::GemHelper.install_tasks
-
-task :default => [:test]
-Rake::TestTask.new(:test) do |t|
-  t.libs << 'lib' << 'test'
-  t.pattern = 'test/**/test_*.rb'
-  t.verbose = true
+# ------------------------------------------------------------
+# RSpec
+
+require 'rspec/core'
+require 'rspec/core/rake_task'
+
+namespace :spec do
+
+  desc 'Run all unit tests'
+  RSpec::Core::RakeTask.new(:unit) do |task|
+    task.rspec_opts = %w[--color --format documentation --order default]
+    task.pattern = 'unit/**/*_spec.rb'
+  end
+
+  task all: [:unit]
 end
 
-RDoc::Task.new do |rd|
-  rd.title = "Merritt Ingest Client"
-  rd.rdoc_files.include("README", "lib/**/*.rb")
+desc 'Run all tests'
+task spec: 'spec:all'
+
+# ------------------------------------------------------------
+# Coverage
 
-  rd.options += ['-f', 'darkfish',]
+desc 'Run all unit tests with coverage'
+task :coverage do
+  ENV['COVERAGE'] = 'true'
+  Rake::Task['spec:unit'].execute
 end
+
+# ------------------------------------------------------------
+# RuboCop
+
+require 'rubocop/rake_task'
+RuboCop::RakeTask.new
+
+# ------------------------------------------------------------
+# Defaults
+
+desc 'Run unit tests, check test coverage, run acceptance tests, check code style'
+task default: %i[coverage rubocop]

data/lib/mrt/ingest.rb

@@ -1,16 +1 @@
-# Author::    Erik Hetzner  (mailto:erik.hetzner@ucop.edu)
-# Copyright:: Copyright (c) 2011, Regents of the University of California
-
-module Mrt
-  module Ingest
-    autoload :Client, "mrt/ingest/client"
-    autoload :IObject, "mrt/ingest/iobject"
-    autoload :MessageDigest, "mrt/ingest/message_digest"
-    autoload :OneTimeServer, "mrt/ingest/one_time_server"
-    autoload :Request, "mrt/ingest/request"
-    autoload :Response, "mrt/ingest/response"
-
-    class IngestException < Exception
-    end
-  end
-end
+Dir.glob(File.expand_path('ingest/*.rb', __dir__)).sort.each(&method(:require))

data/lib/mrt/ingest/client.rb

@@ -10,7 +10,7 @@ module Mrt
 
     # A client for ingesting objects into a Merritt.
     class Client
-      def initialize(base_uri, username=nil, password=nil)
+      def initialize(base_uri, username = nil, password = nil)
         @base_uri = base_uri
         @username = username
         @password = password
@@ -18,19 +18,20 @@ module Mrt
 
       # Send a request to the client.
       def ingest(ingest_req)
-        return Response.new(mk_rest_request(ingest_req).execute())
+        Response.new(mk_rest_request(ingest_req).execute)
       end
 
       # :nodoc:
       def mk_rest_request(ingest_req)
-        args = { 
-          :method   => :post,
-          :url      => @base_uri,
-          :user     => @username,
-          :password => @password,
-          :payload  => ingest_req.mk_args(),
-          :headers  => { :multipart => true } }.delete_if { |k,v| (v.nil? || v == "") }
-        return RestClient::Request.new(args)
+        args = {
+          method: :post,
+          url: @base_uri,
+          user: @username,
+          password: @password,
+          payload: ingest_req.mk_args,
+          headers: { multipart: true }
+        }.delete_if { |_k, v| (v.nil? || v == '') }
+        RestClient::Request.new(args)
       end
 
     end

data/lib/mrt/ingest/component.rb

@@ -0,0 +1,78 @@
+require 'digest/md5'
+
+module Mrt
+  module Ingest
+    # Represents a component of an object to ingest. Either a #URI or a
+    # #File.
+    class Component # :nodoc:
+
+      attr_reader :server, :uri
+
+      def initialize(server, location, options)
+        @server = server
+        @name = options[:name]
+        @digest = options[:digest]
+        @mime_type = options[:mime_type]
+        @size = options[:size]
+
+        init_uri(location)
+      end
+
+      class << self
+        def from_erc(server, erc)
+          return Component.new(server, erc, name: 'mrt-erc.txt') if erc.is_a?(URI) || erc.is_a?(File)
+          return from_hash(server, erc) if erc.is_a?(Hash)
+
+          raise ArgumentError, 'Bad ERC supplied: must be a URI, File, or Hash'
+        end
+
+        def from_hash(server, erc_h)
+          uri_str, path = server.add_file do |f|
+            f.write("erc:\n")
+            erc_h.each_pair { |k, v| f.write("#{k}: #{v}\n") }
+          end
+
+          digest = Mrt::Ingest::MessageDigest::MD5.from_file(File.new(path))
+          Component.new(server, URI.parse(uri_str), name: 'mrt-erc.txt', digest: digest)
+        end
+      end
+
+      def to_manifest_entry
+        "#{@uri} | #{digest_type} | #{digest_value} | #{@size} | | #{@name} | #{@mime_type}\n"
+      end
+
+      def digest_type
+        @digest && @digest.type
+      end
+
+      def digest_value
+        @digest && @digest.value
+      end
+
+      private
+
+      def file_like?(location)
+        [File, Tempfile].any? { |c| location.is_a?(c) }
+      end
+
+      def init_uri(location)
+        return init_from_file(location) if file_like?(location)
+        return init_from_uri(location) if location.is_a?(URI)
+
+        raise ArgumentError, "Trying to add a component that is not a File or URI: #{location}"
+      end
+
+      def init_from_file(file)
+        @name = File.basename(file.path) if @name.nil?
+        @uri = server.add_file(file)[0]
+        @digest = Mrt::Ingest::MessageDigest::MD5.from_file(file) if @digest.nil?
+        @size = File.size(file.path) if @size.nil?
+      end
+
+      def init_from_uri(uri)
+        @name = File.basename(uri.to_s) if @name.nil?
+        @uri = uri
+      end
+    end
+  end
+end