sdr-client 0.15.2 → 0.17.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 4fbf18f89def752a295fe6004aef3a08aed6809d023041aad877619a8309865f
-  data.tar.gz: 608e8c9670b3c929f8a7639c20ed34f4b924f561aed295a42162620ef007d146
+  metadata.gz: 49708bd983a0e4e3322f7a1449d18bd2836f42020fb0c4fec06cf6ce25ec78a5
+  data.tar.gz: a6aecda31d1791026f321147514f7d1ec36a5080aa0df3e4e7638c10125de1f5
 SHA512:
-  metadata.gz: 65633ede9a98646d7e722a6b7d42c761c2a8bc04aec21010ba7187ccfdd3fc5ebd79781140f566b0733a2cb9b87c1f0d14f2cf05de3a483dc45d2ea9fa3c6ed5
-  data.tar.gz: 752f5ce61da771161af8e66e7e0f2ad98ba9e2b6cdf406aa4f82cbf35d9b3ca77926fc5a3a4615393abe67aebd0368342f93145b6137b0f212e9611c39fd5c4c
+  metadata.gz: 3238ad6cbb4ade491ff443448b46f9ee78d081b233a07678046421b1de2a80108074428ccb6a8a768d4cbf1af4a60b14dfb37b8336c83ded75dd04c2cb8a743d
+  data.tar.gz: cf082df7b2efe67c88349644545495d3555206efb6aca5a2eca156ab144db72a509c9a0c5aa1b69a7d71504c8c14bca9fa75b57dcc5c7cd37a00902387d0f73d

data/.rubocop.yml

@@ -13,6 +13,7 @@ Layout/LineLength:
 Metrics/BlockLength:
   Exclude:
     - 'spec/**/*'
+    -  'sdr-client.gemspec'
   ExcludedMethods:
     - 'OptionParser.new'
 

data/README.md

@@ -8,6 +8,8 @@
 This is a CLI for interacting with the Stanford Digital Repository API.
 The code for the SDR API server is at https://github.com/sul-dlss/sdr-api
 
+This provides a way for consumers to easily and correctly deposit files to the SDR without requiring access to the `/dor` NFS mount or to use Hydrus.  A primary design goal was for this to have as few dependencies as possible so that it can be easily distributed by `gem install sdr-client` and then it can be used as a CLI.
+
 ## Install
 
 `gem install sdr-client`

data/lib/sdr_client.rb

@@ -2,9 +2,12 @@
 
 require 'dry/monads'
 require 'faraday'
+require 'active_support'
+require 'active_support/core_ext/object/json'
 
 require 'sdr_client/version'
 require 'sdr_client/deposit'
+require 'sdr_client/model_deposit'
 require 'sdr_client/credentials'
 require 'sdr_client/login'
 require 'sdr_client/login_prompt'

data/lib/sdr_client/deposit.rb

@@ -5,10 +5,11 @@ require 'logger'
 module SdrClient
   # The namespace for the "deposit" command
   module Deposit
+    BOOK_TYPE = 'http://cocina.sul.stanford.edu/models/book.jsonld'
     # rubocop:disable Metrics/ParameterLists
     # rubocop:disable Metrics/MethodLength
     def self.run(label: nil,
-                 type: 'http://cocina.sul.stanford.edu/models/book.jsonld',
+                 type: BOOK_TYPE,
                  viewing_direction: nil,
                  access: 'dark',
                  use_statement: nil,
@@ -26,6 +27,7 @@ module SdrClient
                  logger: Logger.new(STDOUT))
       token = Credentials.read
 
+      augmented_metadata = FileMetadataBuilder.build(files: files, files_metadata: files_metadata)
       metadata = Request.new(label: label,
                              type: type,
                              access: access,
@@ -38,7 +40,7 @@ module SdrClient
                              embargo_release_date: embargo_release_date,
                              embargo_access: embargo_access,
                              viewing_direction: viewing_direction,
-                             files_metadata: files_metadata)
+                             files_metadata: augmented_metadata)
       Process.new(metadata: metadata, url: url, token: token, files: files,
                   grouping_strategy: grouping_strategy, logger: logger).run
     end
@@ -52,6 +54,7 @@ require 'sdr_client/deposit/matching_file_grouping_strategy'
 require 'sdr_client/deposit/files/direct_upload_request'
 require 'sdr_client/deposit/files/direct_upload_response'
 require 'sdr_client/deposit/file'
+require 'sdr_client/deposit/file_metadata_builder'
 require 'sdr_client/deposit/file_set'
 require 'sdr_client/deposit/request'
 require 'sdr_client/deposit/upload_files'

data/lib/sdr_client/deposit/file_metadata_builder.rb

@@ -0,0 +1,51 @@
+# frozen_string_literal: true
+
+require 'sdr_client/deposit/file_metadata_builder_operations/mime_type'
+require 'sdr_client/deposit/file_metadata_builder_operations/md5'
+require 'sdr_client/deposit/file_metadata_builder_operations/sha1'
+
+module SdrClient
+  module Deposit
+    # Build basic metadata for files, iterating over a series of operations
+    # The available options are here: https://github.com/sul-dlss/sdr-client/blob/v0.8.1/lib/sdr_client/deposit/file.rb#L8-L10
+    class FileMetadataBuilder
+      OPERATIONS = [
+        FileMetadataBuilderOperations::MimeType,
+        FileMetadataBuilderOperations::MD5,
+        FileMetadataBuilderOperations::SHA1
+      ].freeze
+      private_constant :OPERATIONS
+
+      # @param (see #initialize)
+      # @return (see #build)
+      def self.build(files:, files_metadata:)
+        new(files: files, files_metadata: files_metadata.dup).build
+      end
+
+      # @param [Array<String>] files the list of files for which to generate metadata
+      def initialize(files:, files_metadata:)
+        @files = files
+        @files_metadata = files_metadata
+      end
+
+      # @return [Hash<String, Hash<String, String>>]
+      def build
+        files.each do |file_path|
+          file_key = ::File.basename(file_path)
+          OPERATIONS.each do |operation|
+            result = operation.for(file_path: file_path)
+            next if result.nil?
+
+            files_metadata[file_key] ||= {}
+            files_metadata[file_key][operation::NAME] = result
+          end
+        end
+        files_metadata
+      end
+
+      private
+
+      attr_reader :files, :files_metadata
+    end
+  end
+end

data/lib/sdr_client/deposit/file_metadata_builder_operations/md5.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+require 'digest'
+
+module SdrClient
+  module Deposit
+    module FileMetadataBuilderOperations
+      # MD5 for this file.
+      class MD5
+        NAME = 'md5'
+        def self.for(file_path:, **)
+          Digest::MD5.file(file_path).hexdigest
+        end
+      end
+    end
+  end
+end

data/lib/sdr_client/deposit/file_metadata_builder_operations/mime_type.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+require 'digest'
+
+module SdrClient
+  module Deposit
+    module FileMetadataBuilderOperations
+      # Mime-type for this file.
+      class MimeType
+        NAME = 'mime_type'
+        def self.for(file_path:, **)
+          `file --mime-type -b #{file_path}`.chomp
+        end
+      end
+    end
+  end
+end

data/lib/sdr_client/deposit/file_metadata_builder_operations/sha1.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+require 'digest'
+
+module SdrClient
+  module Deposit
+    module FileMetadataBuilderOperations
+      # SHA1 for this file.
+      class SHA1
+        NAME = 'sha1'
+        def self.for(file_path:, **)
+          Digest::SHA1.file(file_path).hexdigest
+        end
+      end
+    end
+  end
+end

data/lib/sdr_client/deposit/metadata_builder.rb

@@ -16,6 +16,8 @@ module SdrClient
         @grouping_strategy = grouping_strategy
       end
 
+      # @param [UploadFiles] upload_responses the uploaded file information
+      # @return [Request] the metadata with fileset information added in.
       def with_uploads(upload_responses)
         file_sets = build_filesets(uploads: upload_responses)
         metadata.with_file_sets(file_sets)
@@ -30,9 +32,25 @@ module SdrClient
       def build_filesets(uploads:)
         grouped_uploads = grouping_strategy.run(uploads: uploads)
         grouped_uploads.map.with_index(1) do |upload_group, i|
-          metadata_group = {}
-          upload_group.each { |upload| metadata_group[upload.filename] = metadata.for(upload.filename) }
-          FileSet.new(uploads: upload_group, uploads_metadata: metadata_group, label: "Object #{i}")
+          FileSet.new(uploads: upload_group,
+                      uploads_metadata: metadata_group(upload_group),
+                      label: label(i))
+        end
+      end
+
+      def label(index)
+        case metadata.type
+        when BOOK_TYPE
+          "Page #{index}"
+        else
+          "Object #{index}"
+        end
+      end
+
+      # Get the metadata for the files belonging to a fileset
+      def metadata_group(upload_group)
+        upload_group.each_with_object({}) do |upload, obj|
+          obj[upload.filename] = metadata.for(upload.filename)
         end
       end
     end

data/lib/sdr_client/deposit/model_process.rb

@@ -0,0 +1,124 @@
+# frozen_string_literal: true
+
+require 'logger'
+
+module SdrClient
+  module Deposit
+    # The process for doing a deposit from a Cocina Model
+    class ModelProcess
+      DRO_PATH = '/v1/resources'
+      # @param [Cocina::Model::RequestDRO] request_dro for depositing
+      # @param [String] url the server to send to
+      # @param [String] token the bearer auth token for the server
+      # @param [Array<String>] files a list of file names to upload
+      # @param [Logger] logger the logger to use
+      def initialize(request_dro:, url:,
+                     token:, files: [], logger: Logger.new(STDOUT))
+        @files = files
+        @url = url
+        @token = token
+        @request_dro = request_dro
+        @logger = logger
+      end
+
+      def run
+        check_files_exist
+        child_files_match
+
+        upload_responses = UploadFiles.new(files: files,
+                                           logger: logger,
+                                           connection: connection,
+                                           mime_types: mime_types).run
+        new_request_dro = with_external_identifiers(upload_responses)
+        upload_request_dro(new_request_dro.to_json)
+      end
+
+      private
+
+      attr_reader :request_dro, :files, :url, :token, :logger
+
+      def check_files_exist
+        logger.info('checking to see if files exist')
+        files.each do |file_name|
+          raise Errno::ENOENT, file_name unless ::File.exist?(file_name)
+        end
+      end
+
+      def child_files_match
+        # Files without request files.
+        files.each do |filepath|
+          filename = ::File.basename(filepath)
+
+          raise "Request file not provided for #{filepath}" if request_files[filename].nil?
+        end
+
+        # Request files without files
+        filenames = files.map { |filepath| ::File.basename(filepath) }
+        request_files.keys.each do |request_filename|
+          raise "File not provided for request file #{request_filename}" unless filenames.include?(request_filename)
+        end
+      end
+
+      # @return [Hash<Symbol,String>] the result of the metadata call
+      def upload_request_dro(request_json)
+        logger.info("Starting upload metadata: #{request_json}")
+        response = connection.post(DRO_PATH, request_json, 'Content-Type' => 'application/json')
+        unexpected_response(response) unless response.status == 201
+
+        logger.info("Response from server: #{response.body}")
+
+        { druid: JSON.parse(response.body)['druid'], background_job: response.headers['Location'] }
+      end
+
+      def unexpected_response(response)
+        raise "There was an error with your request: #{response.body}" if response.status == 400
+        raise 'There was an error with your credentials. Perhaps they have expired?' if response.status == 401
+
+        raise "unexpected response: #{response.status} #{response.body}"
+      end
+
+      def connection
+        @connection ||= Faraday.new(url: url) do |conn|
+          conn.authorization :Bearer, token
+          conn.adapter :net_http
+        end
+      end
+
+      # Map of filenames to mimetypes
+      def mime_types
+        @mime_types ||=
+          Hash[
+            request_files.map do |filename, file|
+              [filename, file.hasMimeType || 'application/octet-stream']
+            end
+          ]
+      end
+
+      # Map of filenames to request files
+      def request_files
+        @request_files ||=
+          Hash[
+              request_dro.structural.contains.map do |file_set|
+                file_set.structural.contains.map do |file|
+                  [file.filename, file]
+                end
+              end.flatten(1)
+          ]
+      end
+
+      def with_external_identifiers(upload_responses)
+        signed_id_map = Hash[upload_responses.map { |response| [response.filename, response.signed_id] }]
+
+        # Manipulating request_dro as hash since immutable
+        request_dro_hash = request_dro.to_h
+        request_dro_hash[:structural][:contains].each do |file_set|
+          file_set[:structural][:contains].each do |file|
+            file[:externalIdentifier] = signed_id_map[file[:filename]]
+          end
+        end
+
+        Cocina::Models::RequestDRO.new(request_dro_hash)
+      end
+    end
+  end
+end

data/lib/sdr_client/deposit/process.rb

@@ -30,7 +30,7 @@ module SdrClient
         upload_responses = UploadFiles.new(files: files,
                                            logger: logger,
                                            connection: connection,
-                                           metadata: metadata).run
+                                           mime_types: mime_types).run
         metadata_builder = MetadataBuilder.new(metadata: metadata,
                                                grouping_strategy: grouping_strategy,
                                                logger: logger)
@@ -74,6 +74,16 @@ module SdrClient
           conn.adapter :net_http
         end
       end
+
+      def mime_types
+        @mime_types ||=
+          Hash[
+            files.map do |filepath|
+              filename = ::File.basename(filepath)
+              [filename, metadata.for(filename)['mime_type']]
+            end
+          ]
+      end
     end
   end
 end

data/lib/sdr_client/deposit/request.rb

@@ -79,10 +79,12 @@ module SdrClient
         files_metadata.fetch(filename, {})
       end
 
+      attr_reader :type
+
       private
 
       attr_reader :access, :label, :file_sets, :source_id, :catkey, :apo, :collection,
-                  :type, :files_metadata, :embargo_release_date, :embargo_access,
+                  :files_metadata, :embargo_release_date, :embargo_access,
                   :viewing_direction, :use_statement, :copyright
 
       def administrative

data/lib/sdr_client/deposit/upload_files.rb

@@ -7,13 +7,13 @@ module SdrClient
     # The file uploading part of a deposit
     class UploadFiles
       BLOB_PATH = '/v1/direct_uploads'
-      # @param [Array<String>] files a list of file names to upload
+      # @param [Array<String>] files a list of filepaths to upload
       # @param [Logger] logger the logger to use
       # @param [Faraday::Connection] connection
-      # @param [Request] metadata information about the object
-      def initialize(files:, metadata:, logger:, connection:)
+      # @param [Hash<String,String] mime_types a map of filenames to mime types
+      def initialize(files:, mime_types:, logger:, connection:)
         @files = files
-        @metadata = metadata
+        @mime_types = mime_types
         @logger = logger
         @connection = connection
       end
@@ -28,14 +28,14 @@ module SdrClient
 
       private
 
-      attr_reader :files, :metadata, :logger, :connection
+      attr_reader :files, :mime_types, :logger, :connection
 
       def collect_file_metadata
         files.each_with_object({}) do |path, obj|
           file_name = ::File.basename(path)
           obj[path] = Files::DirectUploadRequest.from_file(path,
                                                            file_name: file_name,
-                                                           content_type: metadata.for(file_name)['mime_type'])
+                                                           content_type: mime_types[file_name])
         end
       end
 

data/lib/sdr_client/model_deposit.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+require 'logger'
+
+module SdrClient
+  # The namespace for the "deposit" command
+  module Deposit
+    def self.model_run(request_dro:,
+                       files: [],
+                       url:,
+                       logger: Logger.new(STDOUT))
+      token = Credentials.read
+
+      ModelProcess.new(request_dro: request_dro, url: url, token: token, files: files, logger: logger).run
+    end
+  end
+end
+require 'sdr_client/deposit/model_process'

data/lib/sdr_client/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module SdrClient
-  VERSION = '0.15.2'
+  VERSION = '0.17.2'
 end

data/sdr-client.gemspec

@@ -27,6 +27,8 @@ Gem::Specification.new do |spec|
   spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
   spec.require_paths = ['lib']
 
+  spec.add_dependency 'activesupport'
+  spec.add_dependency 'cocina-models', '~> 0.28.0'
   spec.add_dependency 'dry-monads'
   spec.add_dependency 'faraday', '>= 0.16'
 

metadata

@@ -1,15 +1,43 @@
 --- !ruby/object:Gem::Specification
 name: sdr-client
 version: !ruby/object:Gem::Version
-  version: 0.15.2
+  version: 0.17.2
 platform: ruby
 authors:
 - Justin Coyne
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2020-03-04 00:00:00.000000000 Z
+date: 2020-03-11 00:00:00.000000000 Z
 dependencies:
+- !ruby/object:Gem::Dependency
+  name: activesupport
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: cocina-models
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.28.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.28.0
 - !ruby/object:Gem::Dependency
   name: dry-monads
   requirement: !ruby/object:Gem::Requirement
@@ -164,17 +192,23 @@ files:
 - lib/sdr_client/credentials.rb
 - lib/sdr_client/deposit.rb
 - lib/sdr_client/deposit/file.rb
+- lib/sdr_client/deposit/file_metadata_builder.rb
+- lib/sdr_client/deposit/file_metadata_builder_operations/md5.rb
+- lib/sdr_client/deposit/file_metadata_builder_operations/mime_type.rb
+- lib/sdr_client/deposit/file_metadata_builder_operations/sha1.rb
 - lib/sdr_client/deposit/file_set.rb
 - lib/sdr_client/deposit/files/direct_upload_request.rb
 - lib/sdr_client/deposit/files/direct_upload_response.rb
 - lib/sdr_client/deposit/matching_file_grouping_strategy.rb
 - lib/sdr_client/deposit/metadata_builder.rb
+- lib/sdr_client/deposit/model_process.rb
 - lib/sdr_client/deposit/process.rb
 - lib/sdr_client/deposit/request.rb
 - lib/sdr_client/deposit/single_file_grouping_strategy.rb
 - lib/sdr_client/deposit/upload_files.rb
 - lib/sdr_client/login.rb
 - lib/sdr_client/login_prompt.rb
+- lib/sdr_client/model_deposit.rb
 - lib/sdr_client/version.rb
 - sdr-client.gemspec
 homepage: https://github.com/sul-dlss/sdr-client