valkyrie-shrine 1.0.0 → 1.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: a7a482b7fc9181f9af0f7a9b9f4fa4d7cc131c9b30489414799b7a0cc16d2c26
-  data.tar.gz: f20feb6e99e26a0c7e4af2bd39b19f2ad5a90d369b142d699abedbaa23829d64
+  metadata.gz: c3422215c665f0b4a25b59b9d62537318f364e1c5522bd8b352de0f92e7c1a03
+  data.tar.gz: e5e537adc20cb7c9f5ca8f1382148263c9631935136de7ac5f9a5998051c7ee2
 SHA512:
-  metadata.gz: 547028b04de7538b83d9bb1685e3424722691be427c40a3c33545b646c5df99cb1ff01ea7b899c175bb330c0163c4359c5d437e6bda81063ae17e4efcbf03273
-  data.tar.gz: 824399812e4df139b35c13c0d45c0adcf35d5c41528ebd235b8d95ecd8d07d7a0b4ae0bb409bec0d7cefdc288dfae8522c1e6c79a39f36842b2ea6e7e6074623
+  metadata.gz: ba9e4ac128bb0cf59ea04cd825dac307decda1b1ad97b21f4fb46e8373d0714d91be79f31f511cb5bb680db126b2ea28117f62d243510d6bcec1c85e8df6fb77
+  data.tar.gz: 5a6a6d9e534cbf4ef8ec561a2a3c569e93862f9b64d2c9f7236ab54428e0a73b2609862c27b05a9fbf340ef714f28facf11d76a74024753873b7247b9f5ede22

data/README.md

@@ -3,7 +3,7 @@
 [Shrine](http://shrinerb.com/) storage adapter for [Valkyrie](https://github.com/samvera-labs/valkyrie).
 
 [![CircleCI](https://circleci.com/gh/samvera-labs/valkyrie-shrine.svg?style=svg)](https://circleci.com/gh/samvera-labs/valkyrie-shrine)
-[![Coverage Status](https://coveralls.io/repos/github/samvera-labs/valkyrie-shrine/badge.svg?branch=master)](https://coveralls.io/github/samvera-labs/valkyrie-shrine?branch=master)
+![Coverage Status](https://img.shields.io/badge/Coverage-100-brightgreen.svg)
 
 ## Installation
 
@@ -21,8 +21,9 @@ Follow the Valkyrie [README](https://github.com/samvera-labs/valkyrie) to get a
   # config/initializers/valkyrie.rb
   require 'shrine/storage/s3'
   require 'shrine/storage/file_system'
-  require 'valkyrie/storage/shrine/checksum/s3'
-  require 'valkyrie/storage/shrine/checksum/file_system'
+  require 'valkyrie/shrine/checksum/s3'
+  require 'valkyrie/shrine/checksum/file_system'
+  require 'valkyrie/shrine/storage/s3'
 
   Shrine.storages = {
     file: Shrine::Storage::FileSystem.new("public", prefix: "uploads"),
@@ -36,6 +37,18 @@ Follow the Valkyrie [README](https://github.com/samvera-labs/valkyrie) to get a
   Valkyrie::StorageAdapter.register(
     Valkyrie::Storage::Shrine.new(Shrine.storages[:file]), :disk
   )
+
+  s3_options = {
+    access_key_id: s3_access_key,
+    bucket: s3_bucket,
+    endpoint: s3_endpoint,
+    force_path_style: force_path_style,
+    region: s3_region,
+    secret_access_key: s3_secret_key
+  }
+  Valkyrie::StorageAdapter.register(
+    Valkyrie::Storage::VersionedShrine.new(Valkyrie::Shrine::Storage::S3.new(**s3_options)), :versioned_s3
+  )
 ```
 
 Then proceed to configure your application following the [Valkyrie documentation](https://github.com/samvera-labs/valkyrie#sample-configuration-configvalkyrieyml) 

data/circle.yml

@@ -7,6 +7,7 @@ jobs:
           BUNDLER_VERSION: 2.0.1
     steps:
       - checkout
+      - run: apt update -y && apt-get install -y lsof
       - run:
           name: Install Bundler 2.0.1
           command: gem install --no-doc bundler:2.0.1

data/lib/valkyrie/shrine/storage/s3.rb

@@ -0,0 +1,64 @@
+# frozen_string_literal: true
+
+require "shrine/storage/s3"
+
+module Valkyrie
+  module Shrine
+    module Storage
+      class S3 < ::Shrine::Storage::S3
+        # List objects that are starting with a given prefix.
+        # This is helpful for versioned files that have a common file identifier.
+        # Need to make sure to combine with storage prefix.
+        #    list_objects(id_prefix: "some/object/id")
+        # @param id_prefix [String] - object's id that starts with
+        # @return [Array(Aws::S3::Object)]
+        def list_objects(id_prefix:)
+          aws_prefix = [*prefix, id_prefix].join("/")
+          bucket.objects(prefix: aws_prefix)
+        end
+
+        # List objects id's that are starting with a given prefix.
+        # This is helpful for versioned files that have a common file identifier.
+        # Need to make sure to combine with storage prefix.
+        #    list_object_ids(id_prefix: "some/object/id")
+        # @param id_prefix [String] - object's id that starts with
+        # @return [Array(String)]
+        def list_object_ids(id_prefix:)
+          objects = list_objects(id_prefix: id_prefix)
+          object_ids_for(objects)
+        end
+
+        # Move a file to a another location.
+        # @param id [String] - the id of the source file
+        # @param destination_id [String] - the id of the destination file
+        # @param destination_bucket [String] - the bucket name of the destination
+        # @return [String]
+        def move_to(id:, destination_id:, destination_bucket: bucket.name)
+          source_object = Aws::S3::Object.new(bucket.name, object_key(id), client: client)
+          destination_key = "#{destination_bucket}/#{object_key(destination_id)}"
+          source_object.move_to(destination_key)
+          destination_key
+        end
+
+        # Deletes all objects in fewest requests possible.
+        # @see +super#delete_objects(objects)+.
+        # @return [Array(string)] List of object id's that are deleted
+        def delete_objects(objects)
+          super
+          object_ids_for(objects)
+        end
+
+        private
+
+          # @ return list of object id's
+          # @param objects [Array(Aws::S3::Object)]
+          def object_ids_for(objects)
+            keys = objects.map(&:key)
+            return keys if prefix.blank?
+
+            keys.map { |k| k.delete_prefix("#{prefix}/") }
+          end
+      end
+    end
+  end
+end

data/lib/valkyrie/shrine/version.rb

@@ -2,6 +2,6 @@
 
 module Valkyrie
   module Shrine
-    VERSION = "1.0.0"
+    VERSION = "1.1.0"
   end
 end

data/lib/valkyrie/shrine.rb

@@ -5,6 +5,8 @@ require 'valkyrie/storage/shrine'
 require 'valkyrie/shrine/checksum/base'
 require 'valkyrie/shrine/checksum/file_system'
 require 'valkyrie/shrine/checksum/s3'
+require 'valkyrie/shrine/storage/s3'
+require 'valkyrie/storage/versioned_shrine'
 
 module Valkyrie
   module Shrine

data/lib/valkyrie/storage/shrine.rb

@@ -69,16 +69,9 @@ module Valkyrie
       # @raise Valkyrie::Shrine::IntegrityError if #verify_checksum is defined
       #   on the shrine object and the file and result digests do not match
       def upload(file:, original_filename:, resource:, **upload_options)
-        # S3 adapter validates options, so we have to remove this one used in
-        # the shared specs.
-        upload_options.delete(:fake_upload_argument)
         identifier = path_generator.generate(resource: resource, file: file, original_filename: original_filename).to_s
-        shrine.upload(file, identifier, **upload_options)
-        find_by(id: "#{protocol_with_prefix}#{identifier}").tap do |result|
-          if verifier
-            raise Valkyrie::Shrine::IntegrityError unless verifier.verify_checksum(file, result)
-          end
-        end
+
+        upload_file(file: file, identifier: identifier, **upload_options)
       end
 
       # Return the file associated with the given identifier
@@ -88,8 +81,6 @@ module Valkyrie
       def find_by(id:)
         raise Valkyrie::StorageAdapter::FileNotFound unless shrine.exists?(shrine_id_for(id))
         Valkyrie::StorageAdapter::StreamFile.new(id: Valkyrie::ID.new(id.to_s), io: DelayedDownload.new(shrine, shrine_id_for(id)))
-      rescue Aws::S3::Errors::NoSuchKey
-        raise Valkyrie::StorageAdapter::FileNotFound
       end
 
       # @param id [Valkyrie::ID]
@@ -104,8 +95,39 @@ module Valkyrie
         shrine.delete(shrine_id_for(id))
       end
 
+      # @param _feature [Symbol] Feature to test for.
+      # @return [Boolean] true if the adapter supports the given feature
+      def supports?(_feature)
+        false
+      end
+
+      # @return [String] identifier prefix
+      def protocol
+        protocol_with_prefix
+      end
+
       private
 
+        # Upload file with a given shrine object idenfifier (without prefix #protocol_with_prefix)
+        # @param file [IO]
+        # @param original_filename [String]
+        # @param resource [Valkyrie::Resource]
+        # @return [Valkyrie::StorageAdapter::StreamFile]
+        # @raise Valkyrie::Shrine::IntegrityError if #verify_checksum is defined
+        #   on the shrine object and the file and result digests do not match
+        def upload_file(file:, identifier:, **upload_options)
+          # S3 adapter validates options, so we have to remove this one used in
+          # the shared specs.
+          upload_options.delete(:fake_upload_argument)
+
+          shrine.upload(file, identifier, **upload_options)
+          find_by(id: "#{protocol_with_prefix}#{identifier}").tap do |result|
+            if verifier
+              raise Valkyrie::Shrine::IntegrityError unless verifier.verify_checksum(file, result)
+            end
+          end
+        end
+
         def try_to_find_verifier
           class_const = shrine.class.name.split(/::/).last.to_sym
           @verifier = Valkyrie::Shrine::Checksum.const_get(class_const).new if Valkyrie::Shrine::Checksum.const_defined?(class_const)

data/lib/valkyrie/storage/versioned_shrine.rb

@@ -0,0 +1,181 @@
+# frozen_string_literal: true
+
+module Valkyrie
+  module Storage
+    # The VersionedShrine adapter implements versioned storage on S3 that manages versions
+    # through Shrine object id with a timestamp like shrine://[resource_id]/[UUID]/v-[timestamp].
+    #
+    # Example to use VersionedShrine storage adapter:
+    #    shrine_s3_options = {
+    #      access_key_id: s3_access_key,
+    #      bucket: s3_bucket,
+    #      endpoint: s3_endpoint,
+    #      force_path_style: force_path_style,
+    #      region: s3_region,
+    #      secret_access_key: s3_secret_key
+    #    }
+    #    Valkyrie::StorageAdapter.register(
+    #      Valkyrie::Storage::VersionedShrine.new(Valkyrie::Shrine::Storage::S3.new(**shrine_s3_options)),
+    #      :s3_repository
+    #    )
+    class VersionedShrine < Shrine
+      # @param feature [Symbol] Feature to test for.
+      # @return [Boolean] true if the adapter supports the given feature
+      def supports?(feature)
+        feature == :versions || feature == :version_deletion
+      end
+
+      # Retireve all files versions with no deletion marker that are associated from S3.
+      # @param id [Valkyrie::ID]
+      # @return [Array(Valkyrie::StorageAdapter::StreamFile)]
+      def find_versions(id:)
+        version_files(id: id).map { |f| find_by(id: f) }
+      end
+
+      # Retireve all file versions associated with the given identifier from S3
+      # @param id [Valkyrie::ID]
+      # @return [Array(Valkyrie::ID)] - list of file identifiers
+      def version_files(id:)
+        shrine.list_object_ids(id_prefix: shrine_id_for(id))
+              .sort
+              .reverse
+              .map { |v| Valkyrie::ID.new(protocol_with_prefix + v) }
+      end
+
+      # Upload a file via the VersionedShrine storage adapter with a version id assigned.
+      # @param file [IO]
+      # @param original_filename [String]
+      # @param resource [Valkyrie::Resource]
+      # @return [Valkyrie::StorageAdapter::StreamFile]
+      # @raise Valkyrie::Shrine::IntegrityError if #verify_checksum is defined
+      #   on the shrine object and the file and result digests do not match
+      def upload(file:, original_filename:, resource:, **upload_options)
+        identifier = path_generator.generate(resource: resource, file: file, original_filename: original_filename)
+        perform_upload(id: "#{protocol_with_prefix}#{identifier}", file: file, **upload_options)
+      end
+
+      # Upload a new version file
+      # @param id [Valkyrie::ID] ID of the Valkyrie::StorageAdapter::File to version.
+      # @param file [IO]
+      def upload_version(id:, file:, **upload_options)
+        # For backward compatablity with files ingested in the past and we don't have to migrate it to versioned fies.
+        #   If there is a file associated with the given identifier that is not a versioned file,
+        #   simply convert it to a versioned file basing on last_modified time to keep all versioned files consistent.
+        migrate_to_versioned(id: id) if shrine.exists?(shrine_id_for(id))
+
+        perform_upload(id: id, file: file, **upload_options)
+      end
+
+      # Upload a file with a version id assigned.
+      # @param id [Valkyrie::ID] ID of the Valkyrie::StorageAdapter::File to version.
+      # @param file [IO]
+      # @return [Valkyrie::StorageAdapter::StreamFile]
+      # @raise Valkyrie::Shrine::IntegrityError if #verify_checksum is defined
+      def perform_upload(id:, file:, **upload_options)
+        shrine_id = shrine_id_for(VersionId.new(id).new_version)
+        upload_file(file: file, identifier: shrine_id, **upload_options)
+      end
+
+      # Delete the versioned file or delete all versions in S3 associated with the given identifier.
+      # @param id [Valkyrie::ID]
+      # @return [Array(Valkyrie::ID)] - file id's that are deleted.
+      # @raise Valkyrie::StorageAdapter::FileNotFound if nothing is found
+      def delete(id:)
+        objects = objects_to_delete(id)
+        raise Valkyrie::StorageAdapter::FileNotFound if objects.blank?
+
+        shrine.delete_objects(objects)
+              .map { |obj_id| Valkyrie::ID.new(protocol_with_prefix + obj_id) }
+      end
+
+      # Find the file associated with the given version identifier
+      #
+      # Note: we need override it to use the latest version
+      #   so that file characterization and derivative creation can use the latest file uploaded
+      # @param id [Valkyrie::ID]
+      # @return [Valkyrie::StorageAdapter::StreamFile]
+      # @raise Valkyrie::StorageAdapter::FileNotFound if nothing is found
+      def find_by(id:)
+        version_id = resolve_current(id)
+
+        raise Valkyrie::StorageAdapter::FileNotFound unless version_id && shrine.exists?(shrine_id_for(version_id.id))
+        Valkyrie::StorageAdapter::StreamFile.new(id: Valkyrie::ID.new(version_id.base_identifier),
+                                                 io: DelayedDownload.new(shrine, shrine_id_for(version_id.id)),
+                                                 version_id: version_id.id)
+      end
+
+      # @return VersionId A VersionId value that's resolved to a current version,
+      #   so we can access the latest version with a base identifier that is not a version.
+      def resolve_current(id)
+        version_id = VersionId.new(id)
+        return version_id if version_id.versioned?
+        version_files = version_files(id: Valkyrie::ID.new(version_id.base_identifier))
+        return nil if version_files.blank?
+        VersionId.new(Valkyrie::ID.new(version_files.first))
+      end
+
+      # @param id [Valkyrie::ID]
+      # @return [Array(Aws::S3::Object)] list of objects
+      def objects_to_delete(id)
+        shrine_id = shrine_id_for(id)
+        version_id = VersionId.new(id)
+        return shrine.list_objects(id_prefix: shrine_id).to_a unless version_id.versioned?
+
+        shrine.exists?(shrine_id) ? [shrine.object(shrine_id)] : []
+      end
+
+      # Convert a non-versioned file to a version file basing on its last_modified time.
+      # @param id [Valkyrie::ID]
+      # @return [VerrsionId]
+      def migrate_to_versioned(id:)
+        shrine_id = shrine_id_for(id)
+        last_modified = shrine.object(shrine_id).last_modified
+        version_id = VersionId.new(id).new_version(timestamp: last_modified)
+        shrine.move_to(id: shrine_id, destination_id: shrine_id_for(version_id))
+        version_id
+      end
+
+      # A class that holds a version id and methods for knowing things about it.
+      # Examples of version ids in this adapter:
+      #   * shrine://[resource_id]/[uuid]/v-20250429142441274
+      #
+      # @note With '/' as path delimiter for versions like '/v-', there is an issue with MinIO
+      #   that fails to list the version files if a file with a base identifier
+      #   that is not a version exisits along with other versioned files associated with the base identifier.
+      class VersionId
+        VERSION_DELIMITER = "/v-"
+
+        attr_reader :id
+        def initialize(id)
+          @id = id
+        end
+
+        # Create new version identifier basing on the given identifier, which could be the original file identifier like
+        #   shrine://[resource_id]/[uuid], or a version identifier like shrine://[resource_id]/[uuid]/v-20250429142441274.
+        # @param timestamp [Time]
+        # @return [String]
+        def new_version(timestamp: nil)
+          version_timestamp = (timestamp&.utc || Time.now.utc).strftime("%Y%m%d%H%M%S%L")
+          versioned? ? string_id.gsub(version, version_timestamp) : string_id + VERSION_DELIMITER + version_timestamp
+        end
+
+        def versioned?
+          string_id.include?(VERSION_DELIMITER)
+        end
+
+        def version
+          string_id.split(VERSION_DELIMITER).last
+        end
+
+        # @return [String] The base identifier that is not a version
+        def base_identifier
+          string_id.split(VERSION_DELIMITER).first
+        end
+
+        def string_id
+          id.to_s
+        end
+      end
+    end
+  end
+end

data/valkyrie-shrine.gemspec

@@ -29,11 +29,11 @@ Gem::Specification.new do |spec|
 
   spec.add_development_dependency 'bixby', '~> 2.0.0.pre.beta1'
   spec.add_development_dependency 'bundler', '~> 2.0'
-  spec.add_development_dependency 'coveralls', '~> 0.8.3'
   spec.add_development_dependency 'pry'
   spec.add_development_dependency 'pry-byebug'
   spec.add_development_dependency 'rake', '>= 12.3.3'
   spec.add_development_dependency 'rspec', '~> 3.0'
   spec.add_development_dependency 'simplecov'
   spec.add_development_dependency 'actionpack'
+  spec.add_development_dependency 'webmock'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: valkyrie-shrine
 version: !ruby/object:Gem::Version
-  version: 1.0.0
+  version: 1.1.0
 platform: ruby
 authors:
 - Brendan Quinn
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2022-07-11 00:00:00.000000000 Z
+date: 2025-11-11 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: aws-sdk-s3
@@ -86,20 +86,6 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '2.0'
-- !ruby/object:Gem::Dependency
-  name: coveralls
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: 0.8.3
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: 0.8.3
 - !ruby/object:Gem::Dependency
   name: pry
   requirement: !ruby/object:Gem::Requirement
@@ -184,6 +170,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: webmock
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 description:
 email:
 - brendan-quinn@northwestern.edu
@@ -206,8 +206,10 @@ files:
 - lib/valkyrie/shrine/checksum/base.rb
 - lib/valkyrie/shrine/checksum/file_system.rb
 - lib/valkyrie/shrine/checksum/s3.rb
+- lib/valkyrie/shrine/storage/s3.rb
 - lib/valkyrie/shrine/version.rb
 - lib/valkyrie/storage/shrine.rb
+- lib/valkyrie/storage/versioned_shrine.rb
 - valkyrie-shrine.gemspec
 homepage: https://github.com/samvera-labs/valkyrie-shrine
 licenses:
@@ -228,7 +230,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.4
+rubygems_version: 3.4.1
 signing_key:
 specification_version: 4
 summary: Shrine storage adapter for Valkyrie