kithe 2.0.0.pre.alpha2 → 2.0.2

data/lib/kithe/version.rb

@@ -1,3 +1,6 @@
 module Kithe
-  VERSION = '2.0.0-alpha2'
+  # not sure why rubygems turned our alphas into 2.0.0.pre.alpha1, inserting
+  # "pre". We need to do same thing with betas to get version orderings
+  # appropriate.
+  VERSION = '2.0.2'
 end

data/lib/shrine/plugins/kithe_checksum_signatures.rb

@@ -0,0 +1,41 @@
+class Shrine
+  module Plugins
+    # Using the shrine signature and add_metadata plugins, ensure that the shrine standard
+    # digest/checksum signatures are recorded in metadata.
+    #
+    # This plugin is NOT included in Kithe::AssetUploader by default, include it in your
+    # local uploader if desired.
+    #
+    # We want to store md5 and sha1 checksums (legacy compat), as well as
+    # sha512 (more recent digital preservation recommendation: https://ocfl.io/draft/spec/#digests)
+    #
+    # We only calculate them only on promotion action (not cache action), to avoid needlessly
+    # expensive double-computation, and because for direct uploads/backgrounding, we haven't
+    # actually gotten the file in our hands to compute checksums until then anyway.
+    #
+    # the add_metadata plugin's `metadata_method` is used to make md5, sha1, and sha512 methods
+    # available on the Attacher. (They also end up delegated from the Asset model)
+    class KitheChecksumSignatures
+      def self.load_dependencies(uploader, *)
+        uploader.plugin :add_metadata
+        uploader.plugin :signature
+      end
+
+      def self.configure(uploader, opts = {})
+        uploader.class_eval do
+          add_metadata do |io, derivative:nil, **context|
+            if context[:action] != :cache && derivative.nil?
+              {
+                md5: calculate_signature(io, :md5),
+                sha1: calculate_signature(io, :sha1),
+                sha512: calculate_signature(io, :sha512)
+              }
+            end
+          end
+          metadata_method :md5, :sha1, :sha512
+        end
+      end
+    end
+    register_plugin(:kithe_checksum_signatures, KitheChecksumSignatures)
+  end
+end

data/lib/shrine/plugins/kithe_controllable_backgrounding.rb

@@ -0,0 +1,53 @@
+class Shrine
+  module Plugins
+
+    # Set up shrine `backgrounding`, where promotion and deletion can happen in a background job.
+    #
+    # https://shrinerb.com/docs/getting-started#backgrounding
+    # https://shrinerb.com/docs/plugins/backgrounding
+    #
+    # By default, kithe does promotion and deletion in kithe-provided ActiveJob classes.
+    #
+    # But this plugin implements code to let you use kithe_promotion_directives to make them happen
+    # inline instead, or disable them.
+    #
+    #     asset.file_attacher.set_promotion_directives(promote: false)
+    #     asset.file_attacher.set_promotion_directives(promote: :inline)
+    #     asset.file_attacher.set_promotion_directives(promote: "inline")
+    #
+    #     asset.file_attacher.set_promotion_directives(delete: :inline)
+    class KitheControllableBackgrounding
+      def self.load_dependencies(uploader, *)
+        uploader.plugin :backgrounding
+        uploader.plugin :kithe_promotion_directives
+      end
+
+      def self.configure(uploader, options = {})
+
+        # promote using shrine backgrounding, but can be effected by promotion_directives[:promote]
+        uploader::Attacher.promote_block do
+          Kithe::TimingPromotionDirective.new(key: :promote, directives: self.promotion_directives) do |directive|
+            if directive.inline?
+              promote
+            elsif directive.background?
+              # What shrine normally expects for backgrounding, plus promotion_directives
+              Kithe::AssetPromoteJob.perform_later(self.class.name, record.class.name, record.id, name.to_s, file_data, self.promotion_directives)
+            end
+          end
+        end
+
+        uploader::Attacher.destroy_block do
+          Kithe::TimingPromotionDirective.new(key: :delete, directives: self.promotion_directives) do |directive|
+            if directive.inline?
+              destroy
+            elsif directive.background?
+              # What shrine normally expects for backgrounding
+              Kithe::AssetDeleteJob.perform_later(self.class.name, data)
+            end
+          end
+        end
+      end
+    end
+    register_plugin(:kithe_controllable_backgrounding, KitheControllableBackgrounding)
+  end
+end

data/lib/shrine/plugins/kithe_derivative_definitions.rb

@@ -0,0 +1,101 @@
+class Shrine
+  module Plugins
+    class KitheDerivativeDefinitions
+      def self.configure(uploader, *opts)
+        # use Rails class_attribute to conveniently have a class-level place
+        # to store our derivative definitions that are inheritable and overrideable.
+        # We store it on the Attacher class, because that's where shrine
+        # puts derivative processor definitions, so seems appropriate.
+        uploader::Attacher.class_attribute :kithe_derivative_definitions, instance_writer: false, default: []
+
+        # Register our derivative processor, that will create our registered derivatives,
+        # with our custom options.
+        uploader::Attacher.derivatives(:kithe_derivatives) do |original, **options|
+          Kithe::Asset::DerivativeCreator.new(self.class.kithe_derivative_definitions,
+            source_io: original,
+            shrine_attacher: self,
+            only: options[:only],
+            except: options[:except],
+            lazy: options[:lazy]
+          ).call
+        end
+      end
+
+      module AttacherClassMethods
+        # Establish a derivative definition that will be used to create a derivative
+        # when #create_derivatives is called, for instance automatically after promotion.
+        #
+        # The most basic definition consists of a derivative key, and a ruby block that
+        # takes the original file, transforms it, and returns a ruby File or other
+        # (shrine-compatible) IO-like object. It will usually be done inside a custom Asset
+        # class definition.
+        #
+        #     class Asset < Kithe::Asset
+        #       define_derivative :thumbnail do |original_file|
+        #       end
+        #     end
+        #
+        # The original_file passed in will be a ruby File object that is already open for reading. If
+        # you need a local file path for your transformation, just use `original_file.path`.
+        #
+        # The return value can be any IO-like object. If it is a ruby File or Tempfile,
+        # that temporary file will be deleted for you after the derivative has been created. If you
+        # have to make any intermediate files, you are responsible for cleaning them up. Ruby stdlib
+        # Tempfile and Dir.mktmpdir may be useful.
+        #
+        # If in order to do your transformation you need additional information about the original,
+        # just add a `record:` keyword argument to your block, and the Asset object will be passed in:
+        #
+        #     define_derivative :thumbnail do |original_file, record:|
+        #        record.width, record.height, record.content_type # etc
+        #     end
+        #
+        # Derivatives are normally uploaded to the Shrine storage labeled :kithe_derivatives,
+        # but a definition can specify an alternate Shrine storage id. (specified shrine storage key
+        # is applied on derivative creation; if you change it with existing derivatives, they should
+        # remain, and be accessible, where they were created; there is no built-in solution at present
+        # for moving them).
+        #
+        #     define_derivative :thumbnail, storage_key: :my_thumb_storage do |original| # ...
+        #
+        # You can also set `default_create: false` if you want a particular definition not to be
+        # included in a no-arg `asset.create_derivatives` that is normally triggered on asset creation.
+        #
+        # And you can set content_type to either a specific type like `image/jpeg` (or array of such) or a general type
+        # like `image`, if you want to define a derivative generation routine for only certain types.
+        # If multiple blocks for the same key are defined, with different content_type restrictions,
+        # the most specific one will be used.  That is, for a JPG, `image/jpeg` beats `image` beats no restriction.
+        def define_derivative(key, content_type: nil, default_create: true, &block)
+          # Make sure we dup the array to handle sub-classes on class_attribute
+          self.kithe_derivative_definitions = self.kithe_derivative_definitions.dup.push(
+            Kithe::Asset::DerivativeDefinition.new(
+              key: key,
+              content_type: content_type,
+              default_create: default_create,
+              proc: block
+            )
+          ).freeze
+        end
+
+        # Returns all derivative keys registered with a definition, as array of strings
+        def defined_derivative_keys
+          self.kithe_derivative_definitions.collect(&:key).uniq.collect(&:to_s)
+        end
+
+        # If you have a subclass that has inherited derivative definitions, you can
+        # remove them -- only by key, will remove any definitions with that key regardless
+        # of content_type restrictions.
+        #
+        # This could be considered rather bad OO design, you might want to consider
+        # a different class hieararchy where you don't have to do this. But it's here.
+        def remove_derivative_definition!(*keys)
+          keys = keys.collect(&:to_sym)
+          self.kithe_derivative_definitions = self.kithe_derivative_definitions.reject do |defn|
+            keys.include?(defn.key.to_sym)
+          end.freeze
+        end
+      end
+    end
+    register_plugin(:kithe_derivative_definitions, KitheDerivativeDefinitions)
+  end
+end

data/lib/shrine/plugins/kithe_derivatives.rb

@@ -0,0 +1,54 @@
+require 'mini_mime'
+
+class Shrine
+  module Plugins
+    # Includes the Shrine `derivatives` plugin with some configuration, and
+    # extra features. The metadata for shrine derivatives is stored in the same
+    # JSON as the main file.
+    #
+    # * default kithe storage location of :kithe_derivatives
+    #
+    # * nice metadata["filename"] for derivatives, instead of default shrine fairly
+    #   random (filename ends up used by default in content-disposition headers when delivered)
+    #
+    # * Includes kithe_persisted_derivatives with #add_persisted_derivatives
+    #   and #create_persisted_derivatives methods for concurrency-safe
+    #   derivative persisting.
+    #
+    # ## Shrine derivatives references
+    #
+    # https://shrinerb.com/docs/plugins/derivatives
+    # https://shrinerb.com/docs/processing
+    class KitheDerivatives
+      def self.load_dependencies(uploader, *)
+        uploader.plugin :derivatives, storage: -> (derivative) do
+          # default derivatives storage to
+          :kithe_derivatives
+        end
+
+        uploader.plugin :kithe_persisted_derivatives
+        uploader.plugin :kithe_derivative_definitions
+      end
+
+      module InstanceMethods
+
+        # Override to fix "filename" metadata to be something reasonable, regardless
+        # of what if anything was the filename of the IO being attached. shrine S3 will
+        # insist on setting a default content-disposition with this filename.
+        def extract_metadata(io, derivative:nil, **context)
+          result = super
+
+          if derivative && context[:record] && result["mime_type"]
+            extension = MiniMime.lookup_by_content_type(result["mime_type"] || "")&.extension || "bin"
+            result["filename"] = "#{context[:record].friendlier_id}_#{derivative}.#{extension}"
+          end
+
+          result
+        end
+      end
+
+    end
+    register_plugin(:kithe_derivatives, KitheDerivatives)
+  end
+end
+

data/lib/shrine/plugins/kithe_determine_mime_type.rb

@@ -0,0 +1,39 @@
+class Shrine
+  module Plugins
+    # Custom kithe logic for determining mime type, using the shrine mime_type plugin.
+    #
+    # We start out using the `marcel` analyzer.
+    # Marcel analyzer is pure-ruby and fast. It's from Basecamp and is what
+    # ActiveStorage uses. It is very similar to :mimemagic (and uses mimemagic
+    # under the hood), but mimemagic seems not to be maintained with up to date
+    # magic db? https://github.com/minad/mimemagic/pull/66
+    #
+    # But marcel is not able to catch some of our MP3s as audio/mpeg. The
+    # `mediainfo` CLI is, and is one of the tools Harvard FITS uses.
+    # If marcel came up blank, AND we are configured to use mediainfo CLI
+    # (which by default we will be if it's available), we will try
+    # shelling out to mediainfo command line.
+    #
+    # https://github.com/MediaArea/MediaInfo
+    #
+    # Ensure that if mime-type can't be otherwise determined, it is assigned
+    # "application/octet-stream", basically the type for generic binary.
+    class KitheDetermineMimeType
+      def self.load_dependencies(uploader, *)
+        uploader.plugin :determine_mime_type, analyzer: -> (io, analyzers) do
+          mime_type = analyzers[:marcel].call(io)
+
+
+          if Kithe.use_mediainfo && mime_type == "application/octet-stream" || mime_type.blank?
+            mime_type = Kithe::MediainfoAnalyzer.new.call(io)
+          end
+
+          mime_type = "application/octet-stream" if mime_type.blank?
+
+          mime_type
+        end
+      end
+    end
+    register_plugin(:kithe_determine_mime_type, KitheDetermineMimeType)
+  end
+end

data/lib/shrine/plugins/kithe_persisted_derivatives.rb

@@ -0,0 +1,161 @@
+class Shrine
+  module Plugins
+    # Some convenience methods for adding/changing derivatives in
+    # concurrency-safe manner:
+    #
+    #   * Won't make a change if the underlying original has changed
+    #     so doesn't match the one you wanted to remove.
+    #   * Won't over-write changes made concurrently in the db by other processes
+    #   * Will always make sure to clean up any temporary files on all error
+    #     and falure conditions.
+    #
+    # Shrine has some building blocks for this, which we use, but it's a bit tricky
+    # to put them together to be generically reliable, as we think we've done here.
+    #
+    # All these methods will cause your Asset model to be saved, because of how
+    # the shrine atomic helpers work. So these methods will by default raise
+    # a TypeError if your Asset model has any unrelated outstanding changes,
+    # but you can tell it to save anyway with `allow_other_changes: true`.
+    #
+    # ## Shrine references:
+    #
+    # https://shrinerb.com/docs/plugins/derivatives
+    # https://shrinerb.com/docs/processing
+    class KithePersistedDerivatives
+      module AttacherMethods
+        # Like the shrine `add_derivatives` method, but also *persists* the
+        # derivatives (saves to db), in a realiably concurrency-safe way.
+        #
+        # Generally can take any options that shrine `add_derivatives`
+        # can take, including custom `storage` or `metadata` arguments.
+        #
+        # Like shrine add_derivatives, it will assume the files passed in are
+        # temporary, and delete them for you. If you want to disable this behavior:
+        #
+        #     attacher.add_persisted_derivatives({key: io}, delete: false)
+        #
+        # In some cases the derivatives can't be persisted because the underlying
+        # database has changed such that they would not be applicable. In those
+        # cases `false` will be return value, otherwise returns the new derivatives
+        # just as shrine `add_derivatives`
+        #
+        # Because the concurrent-safe persistence method will save the associated model --
+        # and save without ActiveRecord validation -- it is not safe to
+        # add_persisted_derivatives on a model with other unsaved changes. The
+        # method will by default refuse to do so, throwing a TypeError. If you'd
+        # like to force it, pass `allow_other_changes: true` as an argument.
+        #
+        # Also takes care of deleting any replaced derivative files, that are no longer
+        # referenced by the model. Shrine by default does not do this:
+        # https://github.com/shrinerb/shrine/issues/468
+        #
+        # All deletions are inline. In general this could be a fairly expensive operation,
+        # it can be wise to do it in a bg job.
+        def add_persisted_derivatives(local_files, **options)
+          other_changes_allowed = !!options.delete(:allow_other_changes)
+          if record && !other_changes_allowed && record.changed?
+            raise TypeError.new("Can't safely add_persisted_derivatives on model with unsaved changes. Pass `allow_other_changes: true` to force.")
+          end
+
+          existing_derivative_files = nil
+
+          # upload to storage
+          new_derivatives = upload_derivatives(local_files, **options)
+
+          begin
+            atomic_persist do |reloaded_attacher|
+              # record so we can delete any replaced ones...
+              existing_derivative_files = map_derivative(reloaded_attacher.derivatives).collect { |path, file| file }
+
+              # make sure we don't override derivatives created in other jobs, by
+              # first using the current up-to-date derivatives from db,
+              # then merging our changes in on top.
+              set_derivatives(reloaded_attacher.derivatives)
+              merge_derivatives(new_derivatives)
+            end
+          rescue Shrine::AttachmentChanged, ActiveRecord::RecordNotFound => e
+            # underlying file has changed or model has been deleted, inappropriate
+            # to add the derivatives, we can just silently drop them, but clean
+            # up after ourselves.
+            delete_derivatives(local_files) unless options[:delete] == false
+            delete_derivatives(new_derivatives)
+
+            return false
+          rescue StandardError => e
+            # unexpected error, clean up our files and re-raise
+            delete_derivatives(local_files) unless options[:delete] == false
+            delete_derivatives(new_derivatives)
+            raise e
+          end
+
+          # Take care of deleting from storage any derivatives that were replaced.
+          current_derivative_files = map_derivative(derivatives).collect { |path, file| file }
+          replaced_files = existing_derivative_files - current_derivative_files
+          delete_derivatives(replaced_files)
+
+          new_derivatives
+        end
+
+        # Like the shrine `create_derivatives` method, but persists the created derivatives
+        # to the database in a concurrency-safe way.
+        #
+        # Can take all options that shrine `create_derivatives` can take, including custom
+        # processors, custom storage key, and arbitrary custom processor arguments.
+        #
+        #     asset.file_attacher.create_persisted_derivatives
+        #     asset.file_attacher.create_persisted_derivatives(storage: :custom_key)
+        #     asset.file_attacher.create_persisted_derivatives(:kithe_derivatives)
+        #     asset.file_attacher.create_persisted_derivatives(:kithe_derivatives, some_arg: "value")
+        #     asset.file_attacher.create_persisted_derivatives(:kithe_derivatives, alternate_source_file)
+        #
+        # Also has an `allow_other_changes` argument, see #add_persisted_derivatives.
+        def create_persisted_derivatives(*args, storage: nil, allow_other_changes: false, **options)
+          return false unless file
+
+          local_files = process_derivatives(*args, **options)
+          add_persisted_derivatives(local_files, storage: storage, allow_other_changes: allow_other_changes)
+        end
+
+        # Kind of like built-in Shrine #remove_derivatives, but also takes care of
+        # persisting AND deleting the removed derivative file from storage --
+        # all in concurrency-safe way, including not making sure to overwrite
+        # any unrelated derivatives someone else was adding.
+        #
+        # Can take the same sorts of path arguments as Shrine derivative #remove_derivatives
+        #
+        #     asset.file_attacher.remove_persisted_derivatives(:small_thumb)
+        #     asset.file_attacher.remove_persisted_derivatives(:small_thumb, :large_thumb)
+        #     asset.file_attacher.remove_persisted_derivatives(:small_thumb, :large_thumb, allow_other_changes: true)
+        def remove_persisted_derivatives(*paths, **options)
+          return if paths.empty?
+
+          other_changes_allowed = !!options.delete(:allow_other_changes)
+          if record && !other_changes_allowed && record.changed?
+            raise TypeError.new("Can't safely add_persisted_derivatives on model with unsaved changes. Pass `allow_other_changes: true` to force.")
+          end
+
+          removed_derivatives = nil
+          atomic_persist do |reloaded_attacher|
+            set_derivatives(reloaded_attacher.derivatives)
+            removed_derivatives = remove_derivatives(*paths, delete: false)
+          end
+
+          if removed_derivatives
+            map_derivative(removed_derivatives) do |_, derivative|
+              derivative.delete if derivative
+            end
+          end
+
+          removed_derivatives
+        rescue Shrine::AttachmentChanged, ActiveRecord::RecordNotFound
+          # original was already deleted or changed, the derivatives wer'e trying to delete.
+          # It should be fine to do nothing, the process that deleted or changed
+          # the model should already have deleted all these derivatives.
+          # But we'll return false as a signel.
+          return false
+        end
+      end
+    end
+    register_plugin(:kithe_persisted_derivatives, KithePersistedDerivatives)
+  end
+end